~hc/RK356X_SDK_RELEASE.git

..	..	@@ -1,3 +1,4 @@
	1	+// SPDX-License-Identifier: GPL-2.0-or-later
1	2	/*
2	3	* Algorithm testing framework and tests.
3	4	*
..	..	@@ -5,6 +6,7 @@
5	6	* Copyright (c) 2002 Jean-Francois Dive <jef@linuxbe.org>
6	7	* Copyright (c) 2007 Nokia Siemens Networks
7	8	* Copyright (c) 2008 Herbert Xu <herbert@gondor.apana.org.au>
	9	+ * Copyright (c) 2019 Google LLC
8	10	*
9	11	* Updated RFC4106 AES-GCM testing.
10	12	* Authors: Aidan O'Mahony (aidan.o.mahony@intel.com)
..	..	@@ -12,12 +14,6 @@
12	14	* Gabriele Paoloni <gabriele.paoloni@intel.com>
13	15	* Tadeusz Struk (tadeusz.struk@intel.com)
14	16	* Copyright (c) 2010, Intel Corporation.
15		- *
16		- * This program is free software; you can redistribute it and/or modify it
17		- * under the terms of the GNU General Public License as published by the Free
18		- * Software Foundation; either version 2 of the License, or (at your option)
19		- * any later version.
20		- *
21	17	*/
22	18
23	19	#include <crypto/aead.h>
..	..	@@ -26,20 +22,43 @@
26	22	#include <linux/err.h>
27	23	#include <linux/fips.h>
28	24	#include <linux/module.h>
	25	+#include <linux/once.h>
	26	+#include <linux/random.h>
29	27	#include <linux/scatterlist.h>
30	28	#include <linux/slab.h>
31	29	#include <linux/string.h>
	30	+#include <linux/uio.h>
32	31	#include <crypto/rng.h>
33	32	#include <crypto/drbg.h>
34	33	#include <crypto/akcipher.h>
35	34	#include <crypto/kpp.h>
36	35	#include <crypto/acompress.h>
	36	+#include <crypto/internal/cipher.h>
	37	+#include <crypto/internal/simd.h>
37	38
38	39	#include "internal.h"
	40	+
	41	+MODULE_IMPORT_NS(CRYPTO_INTERNAL);
39	42
40	43	static bool notests;
41	44	module_param(notests, bool, 0644);
42	45	MODULE_PARM_DESC(notests, "disable crypto self-tests");
	46	+
	47	+static bool panic_on_fail;
	48	+module_param(panic_on_fail, bool, 0444);
	49	+
	50	+#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS
	51	+static bool noextratests;
	52	+module_param(noextratests, bool, 0644);
	53	+MODULE_PARM_DESC(noextratests, "disable expensive crypto self-tests");
	54	+
	55	+static unsigned int fuzz_iterations = 100;
	56	+module_param(fuzz_iterations, uint, 0644);
	57	+MODULE_PARM_DESC(fuzz_iterations, "number of fuzz test iterations");
	58	+
	59	+DEFINE_PER_CPU(bool, crypto_simd_disabled_for_test);
	60	+EXPORT_PER_CPU_SYMBOL_GPL(crypto_simd_disabled_for_test);
	61	+#endif
43	62
44	63	#ifdef CONFIG_CRYPTO_MANAGER_DISABLE_TESTS
45	64
..	..	@@ -59,28 +78,28 @@
59	78	#define XBUFSIZE 8
60	79
61	80	/*
62		- * Indexes into the xbuf to simulate cross-page access.
63		- */
64		-#define IDX1 32
65		-#define IDX2 32400
66		-#define IDX3 1511
67		-#define IDX4 8193
68		-#define IDX5 22222
69		-#define IDX6 17101
70		-#define IDX7 27333
71		-#define IDX8 3000
72		-
73		-/*
74	81	* Used by test_cipher()
75	82	*/
76	83	#define ENCRYPT 1
77	84	#define DECRYPT 0
78	85
79	86	struct aead_test_suite {
80		- struct {
81		- const struct aead_testvec *vecs;
82		- unsigned int count;
83		- } enc, dec;
	87	+ const struct aead_testvec *vecs;
	88	+ unsigned int count;
	89	+
	90	+ /*
	91	+ * Set if trying to decrypt an inauthentic ciphertext with this
	92	+ * algorithm might result in EINVAL rather than EBADMSG, due to other
	93	+ * validation the algorithm does on the inputs such as length checks.
	94	+ */
	95	+ unsigned int einval_allowed : 1;
	96	+
	97	+ /*
	98	+ * Set if this algorithm requires that the IV be located at the end of
	99	+ * the AAD buffer, in addition to being given in the normal way. The
	100	+ * behavior when the two IV copies differ is implementation-defined.
	101	+ */
	102	+ unsigned int aad_iv : 1;
84	103	};
85	104
86	105	struct cipher_test_suite {
..	..	@@ -122,6 +141,7 @@
122	141
123	142	struct alg_test_desc {
124	143	const char *alg;
	144	+ const char *generic_driver;
125	145	int (test)(const struct alg_test_desc desc, const char *driver,
126	146	u32 type, u32 mask);
127	147	int fips_allowed; /* set if alg is allowed in fips mode */
..	..	@@ -138,9 +158,6 @@
138	158	} suite;
139	159	};
140	160
141		-static const unsigned int IDX[8] = {
142		- IDX1, IDX2, IDX3, IDX4, IDX5, IDX6, IDX7, IDX8 };
143		-
144	161	static void hexdump(unsigned char *buf, unsigned int len)
145	162	{
146	163	print_hex_dump(KERN_CONT, "", DUMP_PREFIX_OFFSET,
..	..	@@ -148,12 +165,12 @@
148	165	buf, len, false);
149	166	}
150	167
151		-static int testmgr_alloc_buf(char *buf[XBUFSIZE])
	168	+static int __testmgr_alloc_buf(char *buf[XBUFSIZE], int order)
152	169	{
153	170	int i;
154	171
155	172	for (i = 0; i < XBUFSIZE; i++) {
156		- buf[i] = (void *)__get_free_page(GFP_KERNEL);
	173	+ buf[i] = (char *)__get_free_pages(GFP_KERNEL, order);
157	174	if (!buf[i])
158	175	goto err_free_buf;
159	176	}
..	..	@@ -162,856 +179,2436 @@
162	179
163	180	err_free_buf:
164	181	while (i-- > 0)
165		- free_page((unsigned long)buf[i]);
	182	+ free_pages((unsigned long)buf[i], order);
166	183
167	184	return -ENOMEM;
168	185	}
169	186
170		-static void testmgr_free_buf(char *buf[XBUFSIZE])
	187	+static int testmgr_alloc_buf(char *buf[XBUFSIZE])
	188	+{
	189	+ return __testmgr_alloc_buf(buf, 0);
	190	+}
	191	+
	192	+static void __testmgr_free_buf(char *buf[XBUFSIZE], int order)
171	193	{
172	194	int i;
173	195
174	196	for (i = 0; i < XBUFSIZE; i++)
175		- free_page((unsigned long)buf[i]);
	197	+ free_pages((unsigned long)buf[i], order);
176	198	}
177	199
178		-static int ahash_guard_result(char *result, char c, int size)
	200	+static void testmgr_free_buf(char *buf[XBUFSIZE])
	201	+{
	202	+ __testmgr_free_buf(buf, 0);
	203	+}
	204	+
	205	+#define TESTMGR_POISON_BYTE 0xfe
	206	+#define TESTMGR_POISON_LEN 16
	207	+
	208	+static inline void testmgr_poison(void *addr, size_t len)
	209	+{
	210	+ memset(addr, TESTMGR_POISON_BYTE, len);
	211	+}
	212	+
	213	+/* Is the memory region still fully poisoned? */
	214	+static inline bool testmgr_is_poison(const void *addr, size_t len)
	215	+{
	216	+ return memchr_inv(addr, TESTMGR_POISON_BYTE, len) == NULL;
	217	+}
	218	+
	219	+/* flush type for hash algorithms */
	220	+enum flush_type {
	221	+ /* merge with update of previous buffer(s) */
	222	+ FLUSH_TYPE_NONE = 0,
	223	+
	224	+ /* update with previous buffer(s) before doing this one */
	225	+ FLUSH_TYPE_FLUSH,
	226	+
	227	+ /* likewise, but also export and re-import the intermediate state */
	228	+ FLUSH_TYPE_REIMPORT,
	229	+};
	230	+
	231	+/* finalization function for hash algorithms */
	232	+enum finalization_type {
	233	+ FINALIZATION_TYPE_FINAL, /* use final() */
	234	+ FINALIZATION_TYPE_FINUP, /* use finup() */
	235	+ FINALIZATION_TYPE_DIGEST, /* use digest() */
	236	+};
	237	+
	238	+#define TEST_SG_TOTAL 10000
	239	+
	240	+/**
	241	+ * struct test_sg_division - description of a scatterlist entry
	242	+ *
	243	+ * This struct describes one entry of a scatterlist being constructed to check a
	244	+ * crypto test vector.
	245	+ *
	246	+ * @proportion_of_total: length of this chunk relative to the total length,
	247	+ * given as a proportion out of TEST_SG_TOTAL so that it
	248	+ * scales to fit any test vector
	249	+ * @offset: byte offset into a 2-page buffer at which this chunk will start
	250	+ * @offset_relative_to_alignmask: if true, add the algorithm's alignmask to the
	251	+ * @offset
	252	+ * @flush_type: for hashes, whether an update() should be done now vs.
	253	+ * continuing to accumulate data
	254	+ * @nosimd: if doing the pending update(), do it with SIMD disabled?
	255	+ */
	256	+struct test_sg_division {
	257	+ unsigned int proportion_of_total;
	258	+ unsigned int offset;
	259	+ bool offset_relative_to_alignmask;
	260	+ enum flush_type flush_type;
	261	+ bool nosimd;
	262	+};
	263	+
	264	+/**
	265	+ * struct testvec_config - configuration for testing a crypto test vector
	266	+ *
	267	+ * This struct describes the data layout and other parameters with which each
	268	+ * crypto test vector can be tested.
	269	+ *
	270	+ * @name: name of this config, logged for debugging purposes if a test fails
	271	+ * @inplace: operate on the data in-place, if applicable for the algorithm type?
	272	+ * @req_flags: extra request_flags, e.g. CRYPTO_TFM_REQ_MAY_SLEEP
	273	+ * @src_divs: description of how to arrange the source scatterlist
	274	+ * @dst_divs: description of how to arrange the dst scatterlist, if applicable
	275	+ * for the algorithm type. Defaults to @src_divs if unset.
	276	+ * @iv_offset: misalignment of the IV in the range [0..MAX_ALGAPI_ALIGNMASK+1],
	277	+ * where 0 is aligned to a 2*(MAX_ALGAPI_ALIGNMASK+1) byte boundary
	278	+ * @iv_offset_relative_to_alignmask: if true, add the algorithm's alignmask to
	279	+ * the @iv_offset
	280	+ * @key_offset: misalignment of the key, where 0 is default alignment
	281	+ * @key_offset_relative_to_alignmask: if true, add the algorithm's alignmask to
	282	+ * the @key_offset
	283	+ * @finalization_type: what finalization function to use for hashes
	284	+ * @nosimd: execute with SIMD disabled? Requires !CRYPTO_TFM_REQ_MAY_SLEEP.
	285	+ */
	286	+struct testvec_config {
	287	+ const char *name;
	288	+ bool inplace;
	289	+ u32 req_flags;
	290	+ struct test_sg_division src_divs[XBUFSIZE];
	291	+ struct test_sg_division dst_divs[XBUFSIZE];
	292	+ unsigned int iv_offset;
	293	+ unsigned int key_offset;
	294	+ bool iv_offset_relative_to_alignmask;
	295	+ bool key_offset_relative_to_alignmask;
	296	+ enum finalization_type finalization_type;
	297	+ bool nosimd;
	298	+};
	299	+
	300	+#define TESTVEC_CONFIG_NAMELEN 192
	301	+
	302	+/*
	303	+ * The following are the lists of testvec_configs to test for each algorithm
	304	+ * type when the basic crypto self-tests are enabled, i.e. when
	305	+ * CONFIG_CRYPTO_MANAGER_DISABLE_TESTS is unset. They aim to provide good test
	306	+ * coverage, while keeping the test time much shorter than the full fuzz tests
	307	+ * so that the basic tests can be enabled in a wider range of circumstances.
	308	+ */
	309	+
	310	+/* Configs for skciphers and aeads */
	311	+static const struct testvec_config default_cipher_testvec_configs[] = {
	312	+ {
	313	+ .name = "in-place",
	314	+ .inplace = true,
	315	+ .src_divs = { { .proportion_of_total = 10000 } },
	316	+ }, {
	317	+ .name = "out-of-place",
	318	+ .src_divs = { { .proportion_of_total = 10000 } },
	319	+ }, {
	320	+ .name = "unaligned buffer, offset=1",
	321	+ .src_divs = { { .proportion_of_total = 10000, .offset = 1 } },
	322	+ .iv_offset = 1,
	323	+ .key_offset = 1,
	324	+ }, {
	325	+ .name = "buffer aligned only to alignmask",
	326	+ .src_divs = {
	327	+ {
	328	+ .proportion_of_total = 10000,
	329	+ .offset = 1,
	330	+ .offset_relative_to_alignmask = true,
	331	+ },
	332	+ },
	333	+ .iv_offset = 1,
	334	+ .iv_offset_relative_to_alignmask = true,
	335	+ .key_offset = 1,
	336	+ .key_offset_relative_to_alignmask = true,
	337	+ }, {
	338	+ .name = "two even aligned splits",
	339	+ .src_divs = {
	340	+ { .proportion_of_total = 5000 },
	341	+ { .proportion_of_total = 5000 },
	342	+ },
	343	+ }, {
	344	+ .name = "uneven misaligned splits, may sleep",
	345	+ .req_flags = CRYPTO_TFM_REQ_MAY_SLEEP,
	346	+ .src_divs = {
	347	+ { .proportion_of_total = 1900, .offset = 33 },
	348	+ { .proportion_of_total = 3300, .offset = 7 },
	349	+ { .proportion_of_total = 4800, .offset = 18 },
	350	+ },
	351	+ .iv_offset = 3,
	352	+ .key_offset = 3,
	353	+ }, {
	354	+ .name = "misaligned splits crossing pages, inplace",
	355	+ .inplace = true,
	356	+ .src_divs = {
	357	+ {
	358	+ .proportion_of_total = 7500,
	359	+ .offset = PAGE_SIZE - 32
	360	+ }, {
	361	+ .proportion_of_total = 2500,
	362	+ .offset = PAGE_SIZE - 7
	363	+ },
	364	+ },
	365	+ }
	366	+};
	367	+
	368	+static const struct testvec_config default_hash_testvec_configs[] = {
	369	+ {
	370	+ .name = "init+update+final aligned buffer",
	371	+ .src_divs = { { .proportion_of_total = 10000 } },
	372	+ .finalization_type = FINALIZATION_TYPE_FINAL,
	373	+ }, {
	374	+ .name = "init+finup aligned buffer",
	375	+ .src_divs = { { .proportion_of_total = 10000 } },
	376	+ .finalization_type = FINALIZATION_TYPE_FINUP,
	377	+ }, {
	378	+ .name = "digest aligned buffer",
	379	+ .src_divs = { { .proportion_of_total = 10000 } },
	380	+ .finalization_type = FINALIZATION_TYPE_DIGEST,
	381	+ }, {
	382	+ .name = "init+update+final misaligned buffer",
	383	+ .src_divs = { { .proportion_of_total = 10000, .offset = 1 } },
	384	+ .finalization_type = FINALIZATION_TYPE_FINAL,
	385	+ .key_offset = 1,
	386	+ }, {
	387	+ .name = "digest buffer aligned only to alignmask",
	388	+ .src_divs = {
	389	+ {
	390	+ .proportion_of_total = 10000,
	391	+ .offset = 1,
	392	+ .offset_relative_to_alignmask = true,
	393	+ },
	394	+ },
	395	+ .finalization_type = FINALIZATION_TYPE_DIGEST,
	396	+ .key_offset = 1,
	397	+ .key_offset_relative_to_alignmask = true,
	398	+ }, {
	399	+ .name = "init+update+update+final two even splits",
	400	+ .src_divs = {
	401	+ { .proportion_of_total = 5000 },
	402	+ {
	403	+ .proportion_of_total = 5000,
	404	+ .flush_type = FLUSH_TYPE_FLUSH,
	405	+ },
	406	+ },
	407	+ .finalization_type = FINALIZATION_TYPE_FINAL,
	408	+ }, {
	409	+ .name = "digest uneven misaligned splits, may sleep",
	410	+ .req_flags = CRYPTO_TFM_REQ_MAY_SLEEP,
	411	+ .src_divs = {
	412	+ { .proportion_of_total = 1900, .offset = 33 },
	413	+ { .proportion_of_total = 3300, .offset = 7 },
	414	+ { .proportion_of_total = 4800, .offset = 18 },
	415	+ },
	416	+ .finalization_type = FINALIZATION_TYPE_DIGEST,
	417	+ }, {
	418	+ .name = "digest misaligned splits crossing pages",
	419	+ .src_divs = {
	420	+ {
	421	+ .proportion_of_total = 7500,
	422	+ .offset = PAGE_SIZE - 32,
	423	+ }, {
	424	+ .proportion_of_total = 2500,
	425	+ .offset = PAGE_SIZE - 7,
	426	+ },
	427	+ },
	428	+ .finalization_type = FINALIZATION_TYPE_DIGEST,
	429	+ }, {
	430	+ .name = "import/export",
	431	+ .src_divs = {
	432	+ {
	433	+ .proportion_of_total = 6500,
	434	+ .flush_type = FLUSH_TYPE_REIMPORT,
	435	+ }, {
	436	+ .proportion_of_total = 3500,
	437	+ .flush_type = FLUSH_TYPE_REIMPORT,
	438	+ },
	439	+ },
	440	+ .finalization_type = FINALIZATION_TYPE_FINAL,
	441	+ }
	442	+};
	443	+
	444	+static unsigned int count_test_sg_divisions(const struct test_sg_division *divs)
	445	+{
	446	+ unsigned int remaining = TEST_SG_TOTAL;
	447	+ unsigned int ndivs = 0;
	448	+
	449	+ do {
	450	+ remaining -= divs[ndivs++].proportion_of_total;
	451	+ } while (remaining);
	452	+
	453	+ return ndivs;
	454	+}
	455	+
	456	+#define SGDIVS_HAVE_FLUSHES BIT(0)
	457	+#define SGDIVS_HAVE_NOSIMD BIT(1)
	458	+
	459	+static bool valid_sg_divisions(const struct test_sg_division *divs,
	460	+ unsigned int count, int *flags_ret)
	461	+{
	462	+ unsigned int total = 0;
	463	+ unsigned int i;
	464	+
	465	+ for (i = 0; i < count && total != TEST_SG_TOTAL; i++) {
	466	+ if (divs[i].proportion_of_total <= 0 \|\|
	467	+ divs[i].proportion_of_total > TEST_SG_TOTAL - total)
	468	+ return false;
	469	+ total += divs[i].proportion_of_total;
	470	+ if (divs[i].flush_type != FLUSH_TYPE_NONE)
	471	+ *flags_ret \|= SGDIVS_HAVE_FLUSHES;
	472	+ if (divs[i].nosimd)
	473	+ *flags_ret \|= SGDIVS_HAVE_NOSIMD;
	474	+ }
	475	+ return total == TEST_SG_TOTAL &&
	476	+ memchr_inv(&divs[i], 0, (count - i) * sizeof(divs[0])) == NULL;
	477	+}
	478	+
	479	+/*
	480	+ * Check whether the given testvec_config is valid. This isn't strictly needed
	481	+ * since every testvec_config should be valid, but check anyway so that people
	482	+ * don't unknowingly add broken configs that don't do what they wanted.
	483	+ */
	484	+static bool valid_testvec_config(const struct testvec_config *cfg)
	485	+{
	486	+ int flags = 0;
	487	+
	488	+ if (cfg->name == NULL)
	489	+ return false;
	490	+
	491	+ if (!valid_sg_divisions(cfg->src_divs, ARRAY_SIZE(cfg->src_divs),
	492	+ &flags))
	493	+ return false;
	494	+
	495	+ if (cfg->dst_divs[0].proportion_of_total) {
	496	+ if (!valid_sg_divisions(cfg->dst_divs,
	497	+ ARRAY_SIZE(cfg->dst_divs), &flags))
	498	+ return false;
	499	+ } else {
	500	+ if (memchr_inv(cfg->dst_divs, 0, sizeof(cfg->dst_divs)))
	501	+ return false;
	502	+ /* defaults to dst_divs=src_divs */
	503	+ }
	504	+
	505	+ if (cfg->iv_offset +
	506	+ (cfg->iv_offset_relative_to_alignmask ? MAX_ALGAPI_ALIGNMASK : 0) >
	507	+ MAX_ALGAPI_ALIGNMASK + 1)
	508	+ return false;
	509	+
	510	+ if ((flags & (SGDIVS_HAVE_FLUSHES \| SGDIVS_HAVE_NOSIMD)) &&
	511	+ cfg->finalization_type == FINALIZATION_TYPE_DIGEST)
	512	+ return false;
	513	+
	514	+ if ((cfg->nosimd \|\| (flags & SGDIVS_HAVE_NOSIMD)) &&
	515	+ (cfg->req_flags & CRYPTO_TFM_REQ_MAY_SLEEP))
	516	+ return false;
	517	+
	518	+ return true;
	519	+}
	520	+
	521	+struct test_sglist {
	522	+ char *bufs[XBUFSIZE];
	523	+ struct scatterlist sgl[XBUFSIZE];
	524	+ struct scatterlist sgl_saved[XBUFSIZE];
	525	+ struct scatterlist *sgl_ptr;
	526	+ unsigned int nents;
	527	+};
	528	+
	529	+static int init_test_sglist(struct test_sglist *tsgl)
	530	+{
	531	+ return __testmgr_alloc_buf(tsgl->bufs, 1 /* two pages per buffer */);
	532	+}
	533	+
	534	+static void destroy_test_sglist(struct test_sglist *tsgl)
	535	+{
	536	+ return __testmgr_free_buf(tsgl->bufs, 1 /* two pages per buffer */);
	537	+}
	538	+
	539	+/**
	540	+ * build_test_sglist() - build a scatterlist for a crypto test
	541	+ *
	542	+ * @tsgl: the scatterlist to build. @tsgl->bufs[] contains an array of 2-page
	543	+ * buffers which the scatterlist @tsgl->sgl[] will be made to point into.
	544	+ * @divs: the layout specification on which the scatterlist will be based
	545	+ * @alignmask: the algorithm's alignmask
	546	+ * @total_len: the total length of the scatterlist to build in bytes
	547	+ * @data: if non-NULL, the buffers will be filled with this data until it ends.
	548	+ * Otherwise the buffers will be poisoned. In both cases, some bytes
	549	+ * past the end of each buffer will be poisoned to help detect overruns.
	550	+ * @out_divs: if non-NULL, the test_sg_division to which each scatterlist entry
	551	+ * corresponds will be returned here. This will match @divs except
	552	+ * that divisions resolving to a length of 0 are omitted as they are
	553	+ * not included in the scatterlist.
	554	+ *
	555	+ * Return: 0 or a -errno value
	556	+ */
	557	+static int build_test_sglist(struct test_sglist *tsgl,
	558	+ const struct test_sg_division *divs,
	559	+ const unsigned int alignmask,
	560	+ const unsigned int total_len,
	561	+ struct iov_iter *data,
	562	+ const struct test_sg_division *out_divs[XBUFSIZE])
	563	+{
	564	+ struct {
	565	+ const struct test_sg_division *div;
	566	+ size_t length;
	567	+ } partitions[XBUFSIZE];
	568	+ const unsigned int ndivs = count_test_sg_divisions(divs);
	569	+ unsigned int len_remaining = total_len;
	570	+ unsigned int i;
	571	+
	572	+ BUILD_BUG_ON(ARRAY_SIZE(partitions) != ARRAY_SIZE(tsgl->sgl));
	573	+ if (WARN_ON(ndivs > ARRAY_SIZE(partitions)))
	574	+ return -EINVAL;
	575	+
	576	+ /* Calculate the (div, length) pairs */
	577	+ tsgl->nents = 0;
	578	+ for (i = 0; i < ndivs; i++) {
	579	+ unsigned int len_this_sg =
	580	+ min(len_remaining,
	581	+ (total_len * divs[i].proportion_of_total +
	582	+ TEST_SG_TOTAL / 2) / TEST_SG_TOTAL);
	583	+
	584	+ if (len_this_sg != 0) {
	585	+ partitions[tsgl->nents].div = &divs[i];
	586	+ partitions[tsgl->nents].length = len_this_sg;
	587	+ tsgl->nents++;
	588	+ len_remaining -= len_this_sg;
	589	+ }
	590	+ }
	591	+ if (tsgl->nents == 0) {
	592	+ partitions[tsgl->nents].div = &divs[0];
	593	+ partitions[tsgl->nents].length = 0;
	594	+ tsgl->nents++;
	595	+ }
	596	+ partitions[tsgl->nents - 1].length += len_remaining;
	597	+
	598	+ /* Set up the sgl entries and fill the data or poison */
	599	+ sg_init_table(tsgl->sgl, tsgl->nents);
	600	+ for (i = 0; i < tsgl->nents; i++) {
	601	+ unsigned int offset = partitions[i].div->offset;
	602	+ void *addr;
	603	+
	604	+ if (partitions[i].div->offset_relative_to_alignmask)
	605	+ offset += alignmask;
	606	+
	607	+ while (offset + partitions[i].length + TESTMGR_POISON_LEN >
	608	+ 2 * PAGE_SIZE) {
	609	+ if (WARN_ON(offset <= 0))
	610	+ return -EINVAL;
	611	+ offset /= 2;
	612	+ }
	613	+
	614	+ addr = &tsgl->bufs[i][offset];
	615	+ sg_set_buf(&tsgl->sgl[i], addr, partitions[i].length);
	616	+
	617	+ if (out_divs)
	618	+ out_divs[i] = partitions[i].div;
	619	+
	620	+ if (data) {
	621	+ size_t copy_len, copied;
	622	+
	623	+ copy_len = min(partitions[i].length, data->count);
	624	+ copied = copy_from_iter(addr, copy_len, data);
	625	+ if (WARN_ON(copied != copy_len))
	626	+ return -EINVAL;
	627	+ testmgr_poison(addr + copy_len, partitions[i].length +
	628	+ TESTMGR_POISON_LEN - copy_len);
	629	+ } else {
	630	+ testmgr_poison(addr, partitions[i].length +
	631	+ TESTMGR_POISON_LEN);
	632	+ }
	633	+ }
	634	+
	635	+ sg_mark_end(&tsgl->sgl[tsgl->nents - 1]);
	636	+ tsgl->sgl_ptr = tsgl->sgl;
	637	+ memcpy(tsgl->sgl_saved, tsgl->sgl, tsgl->nents * sizeof(tsgl->sgl[0]));
	638	+ return 0;
	639	+}
	640	+
	641	+/*
	642	+ * Verify that a scatterlist crypto operation produced the correct output.
	643	+ *
	644	+ * @tsgl: scatterlist containing the actual output
	645	+ * @expected_output: buffer containing the expected output
	646	+ * @len_to_check: length of @expected_output in bytes
	647	+ * @unchecked_prefix_len: number of ignored bytes in @tsgl prior to real result
	648	+ * @check_poison: verify that the poison bytes after each chunk are intact?
	649	+ *
	650	+ * Return: 0 if correct, -EINVAL if incorrect, -EOVERFLOW if buffer overrun.
	651	+ */
	652	+static int verify_correct_output(const struct test_sglist *tsgl,
	653	+ const char *expected_output,
	654	+ unsigned int len_to_check,
	655	+ unsigned int unchecked_prefix_len,
	656	+ bool check_poison)
	657	+{
	658	+ unsigned int i;
	659	+
	660	+ for (i = 0; i < tsgl->nents; i++) {
	661	+ struct scatterlist *sg = &tsgl->sgl_ptr[i];
	662	+ unsigned int len = sg->length;
	663	+ unsigned int offset = sg->offset;
	664	+ const char *actual_output;
	665	+
	666	+ if (unchecked_prefix_len) {
	667	+ if (unchecked_prefix_len >= len) {
	668	+ unchecked_prefix_len -= len;
	669	+ continue;
	670	+ }
	671	+ offset += unchecked_prefix_len;
	672	+ len -= unchecked_prefix_len;
	673	+ unchecked_prefix_len = 0;
	674	+ }
	675	+ len = min(len, len_to_check);
	676	+ actual_output = page_address(sg_page(sg)) + offset;
	677	+ if (memcmp(expected_output, actual_output, len) != 0)
	678	+ return -EINVAL;
	679	+ if (check_poison &&
	680	+ !testmgr_is_poison(actual_output + len, TESTMGR_POISON_LEN))
	681	+ return -EOVERFLOW;
	682	+ len_to_check -= len;
	683	+ expected_output += len;
	684	+ }
	685	+ if (WARN_ON(len_to_check != 0))
	686	+ return -EINVAL;
	687	+ return 0;
	688	+}
	689	+
	690	+static bool is_test_sglist_corrupted(const struct test_sglist *tsgl)
	691	+{
	692	+ unsigned int i;
	693	+
	694	+ for (i = 0; i < tsgl->nents; i++) {
	695	+ if (tsgl->sgl[i].page_link != tsgl->sgl_saved[i].page_link)
	696	+ return true;
	697	+ if (tsgl->sgl[i].offset != tsgl->sgl_saved[i].offset)
	698	+ return true;
	699	+ if (tsgl->sgl[i].length != tsgl->sgl_saved[i].length)
	700	+ return true;
	701	+ }
	702	+ return false;
	703	+}
	704	+
	705	+struct cipher_test_sglists {
	706	+ struct test_sglist src;
	707	+ struct test_sglist dst;
	708	+};
	709	+
	710	+static struct cipher_test_sglists *alloc_cipher_test_sglists(void)
	711	+{
	712	+ struct cipher_test_sglists *tsgls;
	713	+
	714	+ tsgls = kmalloc(sizeof(*tsgls), GFP_KERNEL);
	715	+ if (!tsgls)
	716	+ return NULL;
	717	+
	718	+ if (init_test_sglist(&tsgls->src) != 0)
	719	+ goto fail_kfree;
	720	+ if (init_test_sglist(&tsgls->dst) != 0)
	721	+ goto fail_destroy_src;
	722	+
	723	+ return tsgls;
	724	+
	725	+fail_destroy_src:
	726	+ destroy_test_sglist(&tsgls->src);
	727	+fail_kfree:
	728	+ kfree(tsgls);
	729	+ return NULL;
	730	+}
	731	+
	732	+static void free_cipher_test_sglists(struct cipher_test_sglists *tsgls)
	733	+{
	734	+ if (tsgls) {
	735	+ destroy_test_sglist(&tsgls->src);
	736	+ destroy_test_sglist(&tsgls->dst);
	737	+ kfree(tsgls);
	738	+ }
	739	+}
	740	+
	741	+/* Build the src and dst scatterlists for an skcipher or AEAD test */
	742	+static int build_cipher_test_sglists(struct cipher_test_sglists *tsgls,
	743	+ const struct testvec_config *cfg,
	744	+ unsigned int alignmask,
	745	+ unsigned int src_total_len,
	746	+ unsigned int dst_total_len,
	747	+ const struct kvec *inputs,
	748	+ unsigned int nr_inputs)
	749	+{
	750	+ struct iov_iter input;
	751	+ int err;
	752	+
	753	+ iov_iter_kvec(&input, WRITE, inputs, nr_inputs, src_total_len);
	754	+ err = build_test_sglist(&tsgls->src, cfg->src_divs, alignmask,
	755	+ cfg->inplace ?
	756	+ max(dst_total_len, src_total_len) :
	757	+ src_total_len,
	758	+ &input, NULL);
	759	+ if (err)
	760	+ return err;
	761	+
	762	+ if (cfg->inplace) {
	763	+ tsgls->dst.sgl_ptr = tsgls->src.sgl;
	764	+ tsgls->dst.nents = tsgls->src.nents;
	765	+ return 0;
	766	+ }
	767	+ return build_test_sglist(&tsgls->dst,
	768	+ cfg->dst_divs[0].proportion_of_total ?
	769	+ cfg->dst_divs : cfg->src_divs,
	770	+ alignmask, dst_total_len, NULL, NULL);
	771	+}
	772	+
	773	+/*
	774	+ * Support for testing passing a misaligned key to setkey():
	775	+ *
	776	+ * If cfg->key_offset is set, copy the key into a new buffer at that offset,
	777	+ * optionally adding alignmask. Else, just use the key directly.
	778	+ */
	779	+static int prepare_keybuf(const u8 *key, unsigned int ksize,
	780	+ const struct testvec_config *cfg,
	781	+ unsigned int alignmask,
	782	+ const u8 keybuf_ret, const u8 keyptr_ret)
	783	+{
	784	+ unsigned int key_offset = cfg->key_offset;
	785	+ u8 keybuf = NULL, keyptr = (u8 *)key;
	786	+
	787	+ if (key_offset != 0) {
	788	+ if (cfg->key_offset_relative_to_alignmask)
	789	+ key_offset += alignmask;
	790	+ keybuf = kmalloc(key_offset + ksize, GFP_KERNEL);
	791	+ if (!keybuf)
	792	+ return -ENOMEM;
	793	+ keyptr = keybuf + key_offset;
	794	+ memcpy(keyptr, key, ksize);
	795	+ }
	796	+ *keybuf_ret = keybuf;
	797	+ *keyptr_ret = keyptr;
	798	+ return 0;
	799	+}
	800	+
	801	+/* Like setkey_f(tfm, key, ksize), but sometimes misalign the key */
	802	+#define do_setkey(setkey_f, tfm, key, ksize, cfg, alignmask) \
	803	+({ \
	804	+ const u8 keybuf, keyptr; \
	805	+ int err; \
	806	+ \
	807	+ err = prepare_keybuf((key), (ksize), (cfg), (alignmask), \
	808	+ &keybuf, &keyptr); \
	809	+ if (err == 0) { \
	810	+ err = setkey_f((tfm), keyptr, (ksize)); \
	811	+ kfree(keybuf); \
	812	+ } \
	813	+ err; \
	814	+})
	815	+
	816	+#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS
	817	+
	818	+/* Generate a random length in range [0, max_len], but prefer smaller values */
	819	+static unsigned int generate_random_length(unsigned int max_len)
	820	+{
	821	+ unsigned int len = prandom_u32() % (max_len + 1);
	822	+
	823	+ switch (prandom_u32() % 4) {
	824	+ case 0:
	825	+ return len % 64;
	826	+ case 1:
	827	+ return len % 256;
	828	+ case 2:
	829	+ return len % 1024;
	830	+ default:
	831	+ return len;
	832	+ }
	833	+}
	834	+
	835	+/* Flip a random bit in the given nonempty data buffer */
	836	+static void flip_random_bit(u8 *buf, size_t size)
	837	+{
	838	+ size_t bitpos;
	839	+
	840	+ bitpos = prandom_u32() % (size * 8);
	841	+ buf[bitpos / 8] ^= 1 << (bitpos % 8);
	842	+}
	843	+
	844	+/* Flip a random byte in the given nonempty data buffer */
	845	+static void flip_random_byte(u8 *buf, size_t size)
	846	+{
	847	+ buf[prandom_u32() % size] ^= 0xff;
	848	+}
	849	+
	850	+/* Sometimes make some random changes to the given nonempty data buffer */
	851	+static void mutate_buffer(u8 *buf, size_t size)
	852	+{
	853	+ size_t num_flips;
	854	+ size_t i;
	855	+
	856	+ /* Sometimes flip some bits */
	857	+ if (prandom_u32() % 4 == 0) {
	858	+ num_flips = min_t(size_t, 1 << (prandom_u32() % 8), size * 8);
	859	+ for (i = 0; i < num_flips; i++)
	860	+ flip_random_bit(buf, size);
	861	+ }
	862	+
	863	+ /* Sometimes flip some bytes */
	864	+ if (prandom_u32() % 4 == 0) {
	865	+ num_flips = min_t(size_t, 1 << (prandom_u32() % 8), size);
	866	+ for (i = 0; i < num_flips; i++)
	867	+ flip_random_byte(buf, size);
	868	+ }
	869	+}
	870	+
	871	+/* Randomly generate 'count' bytes, but sometimes make them "interesting" */
	872	+static void generate_random_bytes(u8 *buf, size_t count)
	873	+{
	874	+ u8 b;
	875	+ u8 increment;
	876	+ size_t i;
	877	+
	878	+ if (count == 0)
	879	+ return;
	880	+
	881	+ switch (prandom_u32() % 8) { /* Choose a generation strategy */
	882	+ case 0:
	883	+ case 1:
	884	+ /* All the same byte, plus optional mutations */
	885	+ switch (prandom_u32() % 4) {
	886	+ case 0:
	887	+ b = 0x00;
	888	+ break;
	889	+ case 1:
	890	+ b = 0xff;
	891	+ break;
	892	+ default:
	893	+ b = (u8)prandom_u32();
	894	+ break;
	895	+ }
	896	+ memset(buf, b, count);
	897	+ mutate_buffer(buf, count);
	898	+ break;
	899	+ case 2:
	900	+ /* Ascending or descending bytes, plus optional mutations */
	901	+ increment = (u8)prandom_u32();
	902	+ b = (u8)prandom_u32();
	903	+ for (i = 0; i < count; i++, b += increment)
	904	+ buf[i] = b;
	905	+ mutate_buffer(buf, count);
	906	+ break;
	907	+ default:
	908	+ /* Fully random bytes */
	909	+ for (i = 0; i < count; i++)
	910	+ buf[i] = (u8)prandom_u32();
	911	+ }
	912	+}
	913	+
	914	+static char generate_random_sgl_divisions(struct test_sg_division divs,
	915	+ size_t max_divs, char p, char end,
	916	+ bool gen_flushes, u32 req_flags)
	917	+{
	918	+ struct test_sg_division *div = divs;
	919	+ unsigned int remaining = TEST_SG_TOTAL;
	920	+
	921	+ do {
	922	+ unsigned int this_len;
	923	+ const char *flushtype_str;
	924	+
	925	+ if (div == &divs[max_divs - 1] \|\| prandom_u32() % 2 == 0)
	926	+ this_len = remaining;
	927	+ else
	928	+ this_len = 1 + (prandom_u32() % remaining);
	929	+ div->proportion_of_total = this_len;
	930	+
	931	+ if (prandom_u32() % 4 == 0)
	932	+ div->offset = (PAGE_SIZE - 128) + (prandom_u32() % 128);
	933	+ else if (prandom_u32() % 2 == 0)
	934	+ div->offset = prandom_u32() % 32;
	935	+ else
	936	+ div->offset = prandom_u32() % PAGE_SIZE;
	937	+ if (prandom_u32() % 8 == 0)
	938	+ div->offset_relative_to_alignmask = true;
	939	+
	940	+ div->flush_type = FLUSH_TYPE_NONE;
	941	+ if (gen_flushes) {
	942	+ switch (prandom_u32() % 4) {
	943	+ case 0:
	944	+ div->flush_type = FLUSH_TYPE_REIMPORT;
	945	+ break;
	946	+ case 1:
	947	+ div->flush_type = FLUSH_TYPE_FLUSH;
	948	+ break;
	949	+ }
	950	+ }
	951	+
	952	+ if (div->flush_type != FLUSH_TYPE_NONE &&
	953	+ !(req_flags & CRYPTO_TFM_REQ_MAY_SLEEP) &&
	954	+ prandom_u32() % 2 == 0)
	955	+ div->nosimd = true;
	956	+
	957	+ switch (div->flush_type) {
	958	+ case FLUSH_TYPE_FLUSH:
	959	+ if (div->nosimd)
	960	+ flushtype_str = "<flush,nosimd>";
	961	+ else
	962	+ flushtype_str = "<flush>";
	963	+ break;
	964	+ case FLUSH_TYPE_REIMPORT:
	965	+ if (div->nosimd)
	966	+ flushtype_str = "<reimport,nosimd>";
	967	+ else
	968	+ flushtype_str = "<reimport>";
	969	+ break;
	970	+ default:
	971	+ flushtype_str = "";
	972	+ break;
	973	+ }
	974	+
	975	+ BUILD_BUG_ON(TEST_SG_TOTAL != 10000); /* for "%u.%u%%" */
	976	+ p += scnprintf(p, end - p, "%s%u.%u%%@%s+%u%s", flushtype_str,
	977	+ this_len / 100, this_len % 100,
	978	+ div->offset_relative_to_alignmask ?
	979	+ "alignmask" : "",
	980	+ div->offset, this_len == remaining ? "" : ", ");
	981	+ remaining -= this_len;
	982	+ div++;
	983	+ } while (remaining);
	984	+
	985	+ return p;
	986	+}
	987	+
	988	+/* Generate a random testvec_config for fuzz testing */
	989	+static void generate_random_testvec_config(struct testvec_config *cfg,
	990	+ char *name, size_t max_namelen)
	991	+{
	992	+ char *p = name;
	993	+ char * const end = name + max_namelen;
	994	+
	995	+ memset(cfg, 0, sizeof(*cfg));
	996	+
	997	+ cfg->name = name;
	998	+
	999	+ p += scnprintf(p, end - p, "random:");
	1000	+
	1001	+ if (prandom_u32() % 2 == 0) {
	1002	+ cfg->inplace = true;
	1003	+ p += scnprintf(p, end - p, " inplace");
	1004	+ }
	1005	+
	1006	+ if (prandom_u32() % 2 == 0) {
	1007	+ cfg->req_flags \|= CRYPTO_TFM_REQ_MAY_SLEEP;
	1008	+ p += scnprintf(p, end - p, " may_sleep");
	1009	+ }
	1010	+
	1011	+ switch (prandom_u32() % 4) {
	1012	+ case 0:
	1013	+ cfg->finalization_type = FINALIZATION_TYPE_FINAL;
	1014	+ p += scnprintf(p, end - p, " use_final");
	1015	+ break;
	1016	+ case 1:
	1017	+ cfg->finalization_type = FINALIZATION_TYPE_FINUP;
	1018	+ p += scnprintf(p, end - p, " use_finup");
	1019	+ break;
	1020	+ default:
	1021	+ cfg->finalization_type = FINALIZATION_TYPE_DIGEST;
	1022	+ p += scnprintf(p, end - p, " use_digest");
	1023	+ break;
	1024	+ }
	1025	+
	1026	+ if (!(cfg->req_flags & CRYPTO_TFM_REQ_MAY_SLEEP) &&
	1027	+ prandom_u32() % 2 == 0) {
	1028	+ cfg->nosimd = true;
	1029	+ p += scnprintf(p, end - p, " nosimd");
	1030	+ }
	1031	+
	1032	+ p += scnprintf(p, end - p, " src_divs=[");
	1033	+ p = generate_random_sgl_divisions(cfg->src_divs,
	1034	+ ARRAY_SIZE(cfg->src_divs), p, end,
	1035	+ (cfg->finalization_type !=
	1036	+ FINALIZATION_TYPE_DIGEST),
	1037	+ cfg->req_flags);
	1038	+ p += scnprintf(p, end - p, "]");
	1039	+
	1040	+ if (!cfg->inplace && prandom_u32() % 2 == 0) {
	1041	+ p += scnprintf(p, end - p, " dst_divs=[");
	1042	+ p = generate_random_sgl_divisions(cfg->dst_divs,
	1043	+ ARRAY_SIZE(cfg->dst_divs),
	1044	+ p, end, false,
	1045	+ cfg->req_flags);
	1046	+ p += scnprintf(p, end - p, "]");
	1047	+ }
	1048	+
	1049	+ if (prandom_u32() % 2 == 0) {
	1050	+ cfg->iv_offset = 1 + (prandom_u32() % MAX_ALGAPI_ALIGNMASK);
	1051	+ p += scnprintf(p, end - p, " iv_offset=%u", cfg->iv_offset);
	1052	+ }
	1053	+
	1054	+ if (prandom_u32() % 2 == 0) {
	1055	+ cfg->key_offset = 1 + (prandom_u32() % MAX_ALGAPI_ALIGNMASK);
	1056	+ p += scnprintf(p, end - p, " key_offset=%u", cfg->key_offset);
	1057	+ }
	1058	+
	1059	+ WARN_ON_ONCE(!valid_testvec_config(cfg));
	1060	+}
	1061	+
	1062	+static void crypto_disable_simd_for_test(void)
	1063	+{
	1064	+ preempt_disable();
	1065	+ __this_cpu_write(crypto_simd_disabled_for_test, true);
	1066	+}
	1067	+
	1068	+static void crypto_reenable_simd_for_test(void)
	1069	+{
	1070	+ __this_cpu_write(crypto_simd_disabled_for_test, false);
	1071	+ preempt_enable();
	1072	+}
	1073	+
	1074	+/*
	1075	+ * Given an algorithm name, build the name of the generic implementation of that
	1076	+ * algorithm, assuming the usual naming convention. Specifically, this appends
	1077	+ * "-generic" to every part of the name that is not a template name. Examples:
	1078	+ *
	1079	+ * aes => aes-generic
	1080	+ * cbc(aes) => cbc(aes-generic)
	1081	+ * cts(cbc(aes)) => cts(cbc(aes-generic))
	1082	+ * rfc7539(chacha20,poly1305) => rfc7539(chacha20-generic,poly1305-generic)
	1083	+ *
	1084	+ * Return: 0 on success, or -ENAMETOOLONG if the generic name would be too long
	1085	+ */
	1086	+static int build_generic_driver_name(const char *algname,
	1087	+ char driver_name[CRYPTO_MAX_ALG_NAME])
	1088	+{
	1089	+ const char *in = algname;
	1090	+ char *out = driver_name;
	1091	+ size_t len = strlen(algname);
	1092	+
	1093	+ if (len >= CRYPTO_MAX_ALG_NAME)
	1094	+ goto too_long;
	1095	+ do {
	1096	+ const char *in_saved = in;
	1097	+
	1098	+ while (in && in != '(' && in != ')' && in != ',')
	1099	+ out++ = in++;
	1100	+ if (*in != '(' && in > in_saved) {
	1101	+ len += 8;
	1102	+ if (len >= CRYPTO_MAX_ALG_NAME)
	1103	+ goto too_long;
	1104	+ memcpy(out, "-generic", 8);
	1105	+ out += 8;
	1106	+ }
	1107	+ } while ((out++ = in++) != '\0');
	1108	+ return 0;
	1109	+
	1110	+too_long:
	1111	+ pr_err("alg: generic driver name for \"%s\" would be too long\n",
	1112	+ algname);
	1113	+ return -ENAMETOOLONG;
	1114	+}
	1115	+#else /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */
	1116	+static void crypto_disable_simd_for_test(void)
	1117	+{
	1118	+}
	1119	+
	1120	+static void crypto_reenable_simd_for_test(void)
	1121	+{
	1122	+}
	1123	+#endif /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */
	1124	+
	1125	+static int build_hash_sglist(struct test_sglist *tsgl,
	1126	+ const struct hash_testvec *vec,
	1127	+ const struct testvec_config *cfg,
	1128	+ unsigned int alignmask,
	1129	+ const struct test_sg_division *divs[XBUFSIZE])
	1130	+{
	1131	+ struct kvec kv;
	1132	+ struct iov_iter input;
	1133	+
	1134	+ kv.iov_base = (void *)vec->plaintext;
	1135	+ kv.iov_len = vec->psize;
	1136	+ iov_iter_kvec(&input, WRITE, &kv, 1, vec->psize);
	1137	+ return build_test_sglist(tsgl, cfg->src_divs, alignmask, vec->psize,
	1138	+ &input, divs);
	1139	+}
	1140	+
	1141	+static int check_hash_result(const char *type,
	1142	+ const u8 *result, unsigned int digestsize,
	1143	+ const struct hash_testvec *vec,
	1144	+ const char *vec_name,
	1145	+ const char *driver,
	1146	+ const struct testvec_config *cfg)
	1147	+{
	1148	+ if (memcmp(result, vec->digest, digestsize) != 0) {
	1149	+ pr_err("alg: %s: %s test failed (wrong result) on test vector %s, cfg=\"%s\"\n",
	1150	+ type, driver, vec_name, cfg->name);
	1151	+ return -EINVAL;
	1152	+ }
	1153	+ if (!testmgr_is_poison(&result[digestsize], TESTMGR_POISON_LEN)) {
	1154	+ pr_err("alg: %s: %s overran result buffer on test vector %s, cfg=\"%s\"\n",
	1155	+ type, driver, vec_name, cfg->name);
	1156	+ return -EOVERFLOW;
	1157	+ }
	1158	+ return 0;
	1159	+}
	1160	+
	1161	+static inline int check_shash_op(const char *op, int err,
	1162	+ const char driver, const char vec_name,
	1163	+ const struct testvec_config *cfg)
	1164	+{
	1165	+ if (err)
	1166	+ pr_err("alg: shash: %s %s() failed with err %d on test vector %s, cfg=\"%s\"\n",
	1167	+ driver, op, err, vec_name, cfg->name);
	1168	+ return err;
	1169	+}
	1170	+
	1171	+static inline const void sg_data(struct scatterlist sg)
	1172	+{
	1173	+ return page_address(sg_page(sg)) + sg->offset;
	1174	+}
	1175	+
	1176	+/* Test one hash test vector in one configuration, using the shash API */
	1177	+static int test_shash_vec_cfg(const char *driver,
	1178	+ const struct hash_testvec *vec,
	1179	+ const char *vec_name,
	1180	+ const struct testvec_config *cfg,
	1181	+ struct shash_desc *desc,
	1182	+ struct test_sglist *tsgl,
	1183	+ u8 *hashstate)
	1184	+{
	1185	+ struct crypto_shash *tfm = desc->tfm;
	1186	+ const unsigned int alignmask = crypto_shash_alignmask(tfm);
	1187	+ const unsigned int digestsize = crypto_shash_digestsize(tfm);
	1188	+ const unsigned int statesize = crypto_shash_statesize(tfm);
	1189	+ const struct test_sg_division *divs[XBUFSIZE];
	1190	+ unsigned int i;
	1191	+ u8 result[HASH_MAX_DIGESTSIZE + TESTMGR_POISON_LEN];
	1192	+ int err;
	1193	+
	1194	+ /* Set the key, if specified */
	1195	+ if (vec->ksize) {
	1196	+ err = do_setkey(crypto_shash_setkey, tfm, vec->key, vec->ksize,
	1197	+ cfg, alignmask);
	1198	+ if (err) {
	1199	+ if (err == vec->setkey_error)
	1200	+ return 0;
	1201	+ pr_err("alg: shash: %s setkey failed on test vector %s; expected_error=%d, actual_error=%d, flags=%#x\n",
	1202	+ driver, vec_name, vec->setkey_error, err,
	1203	+ crypto_shash_get_flags(tfm));
	1204	+ return err;
	1205	+ }
	1206	+ if (vec->setkey_error) {
	1207	+ pr_err("alg: shash: %s setkey unexpectedly succeeded on test vector %s; expected_error=%d\n",
	1208	+ driver, vec_name, vec->setkey_error);
	1209	+ return -EINVAL;
	1210	+ }
	1211	+ }
	1212	+
	1213	+ /* Build the scatterlist for the source data */
	1214	+ err = build_hash_sglist(tsgl, vec, cfg, alignmask, divs);
	1215	+ if (err) {
	1216	+ pr_err("alg: shash: %s: error preparing scatterlist for test vector %s, cfg=\"%s\"\n",
	1217	+ driver, vec_name, cfg->name);
	1218	+ return err;
	1219	+ }
	1220	+
	1221	+ /* Do the actual hashing */
	1222	+
	1223	+ testmgr_poison(desc->__ctx, crypto_shash_descsize(tfm));
	1224	+ testmgr_poison(result, digestsize + TESTMGR_POISON_LEN);
	1225	+
	1226	+ if (cfg->finalization_type == FINALIZATION_TYPE_DIGEST \|\|
	1227	+ vec->digest_error) {
	1228	+ /* Just using digest() */
	1229	+ if (tsgl->nents != 1)
	1230	+ return 0;
	1231	+ if (cfg->nosimd)
	1232	+ crypto_disable_simd_for_test();
	1233	+ err = crypto_shash_digest(desc, sg_data(&tsgl->sgl[0]),
	1234	+ tsgl->sgl[0].length, result);
	1235	+ if (cfg->nosimd)
	1236	+ crypto_reenable_simd_for_test();
	1237	+ if (err) {
	1238	+ if (err == vec->digest_error)
	1239	+ return 0;
	1240	+ pr_err("alg: shash: %s digest() failed on test vector %s; expected_error=%d, actual_error=%d, cfg=\"%s\"\n",
	1241	+ driver, vec_name, vec->digest_error, err,
	1242	+ cfg->name);
	1243	+ return err;
	1244	+ }
	1245	+ if (vec->digest_error) {
	1246	+ pr_err("alg: shash: %s digest() unexpectedly succeeded on test vector %s; expected_error=%d, cfg=\"%s\"\n",
	1247	+ driver, vec_name, vec->digest_error, cfg->name);
	1248	+ return -EINVAL;
	1249	+ }
	1250	+ goto result_ready;
	1251	+ }
	1252	+
	1253	+ /* Using init(), zero or more update(), then final() or finup() */
	1254	+
	1255	+ if (cfg->nosimd)
	1256	+ crypto_disable_simd_for_test();
	1257	+ err = crypto_shash_init(desc);
	1258	+ if (cfg->nosimd)
	1259	+ crypto_reenable_simd_for_test();
	1260	+ err = check_shash_op("init", err, driver, vec_name, cfg);
	1261	+ if (err)
	1262	+ return err;
	1263	+
	1264	+ for (i = 0; i < tsgl->nents; i++) {
	1265	+ if (i + 1 == tsgl->nents &&
	1266	+ cfg->finalization_type == FINALIZATION_TYPE_FINUP) {
	1267	+ if (divs[i]->nosimd)
	1268	+ crypto_disable_simd_for_test();
	1269	+ err = crypto_shash_finup(desc, sg_data(&tsgl->sgl[i]),
	1270	+ tsgl->sgl[i].length, result);
	1271	+ if (divs[i]->nosimd)
	1272	+ crypto_reenable_simd_for_test();
	1273	+ err = check_shash_op("finup", err, driver, vec_name,
	1274	+ cfg);
	1275	+ if (err)
	1276	+ return err;
	1277	+ goto result_ready;
	1278	+ }
	1279	+ if (divs[i]->nosimd)
	1280	+ crypto_disable_simd_for_test();
	1281	+ err = crypto_shash_update(desc, sg_data(&tsgl->sgl[i]),
	1282	+ tsgl->sgl[i].length);
	1283	+ if (divs[i]->nosimd)
	1284	+ crypto_reenable_simd_for_test();
	1285	+ err = check_shash_op("update", err, driver, vec_name, cfg);
	1286	+ if (err)
	1287	+ return err;
	1288	+ if (divs[i]->flush_type == FLUSH_TYPE_REIMPORT) {
	1289	+ /* Test ->export() and ->import() */
	1290	+ testmgr_poison(hashstate + statesize,
	1291	+ TESTMGR_POISON_LEN);
	1292	+ err = crypto_shash_export(desc, hashstate);
	1293	+ err = check_shash_op("export", err, driver, vec_name,
	1294	+ cfg);
	1295	+ if (err)
	1296	+ return err;
	1297	+ if (!testmgr_is_poison(hashstate + statesize,
	1298	+ TESTMGR_POISON_LEN)) {
	1299	+ pr_err("alg: shash: %s export() overran state buffer on test vector %s, cfg=\"%s\"\n",
	1300	+ driver, vec_name, cfg->name);
	1301	+ return -EOVERFLOW;
	1302	+ }
	1303	+ testmgr_poison(desc->__ctx, crypto_shash_descsize(tfm));
	1304	+ err = crypto_shash_import(desc, hashstate);
	1305	+ err = check_shash_op("import", err, driver, vec_name,
	1306	+ cfg);
	1307	+ if (err)
	1308	+ return err;
	1309	+ }
	1310	+ }
	1311	+
	1312	+ if (cfg->nosimd)
	1313	+ crypto_disable_simd_for_test();
	1314	+ err = crypto_shash_final(desc, result);
	1315	+ if (cfg->nosimd)
	1316	+ crypto_reenable_simd_for_test();
	1317	+ err = check_shash_op("final", err, driver, vec_name, cfg);
	1318	+ if (err)
	1319	+ return err;
	1320	+result_ready:
	1321	+ return check_hash_result("shash", result, digestsize, vec, vec_name,
	1322	+ driver, cfg);
	1323	+}
	1324	+
	1325	+static int do_ahash_op(int (op)(struct ahash_request req),
	1326	+ struct ahash_request *req,
	1327	+ struct crypto_wait *wait, bool nosimd)
	1328	+{
	1329	+ int err;
	1330	+
	1331	+ if (nosimd)
	1332	+ crypto_disable_simd_for_test();
	1333	+
	1334	+ err = op(req);
	1335	+
	1336	+ if (nosimd)
	1337	+ crypto_reenable_simd_for_test();
	1338	+
	1339	+ return crypto_wait_req(err, wait);
	1340	+}
	1341	+
	1342	+static int check_nonfinal_ahash_op(const char *op, int err,
	1343	+ u8 *result, unsigned int digestsize,
	1344	+ const char driver, const char vec_name,
	1345	+ const struct testvec_config *cfg)
	1346	+{
	1347	+ if (err) {
	1348	+ pr_err("alg: ahash: %s %s() failed with err %d on test vector %s, cfg=\"%s\"\n",
	1349	+ driver, op, err, vec_name, cfg->name);
	1350	+ return err;
	1351	+ }
	1352	+ if (!testmgr_is_poison(result, digestsize)) {
	1353	+ pr_err("alg: ahash: %s %s() used result buffer on test vector %s, cfg=\"%s\"\n",
	1354	+ driver, op, vec_name, cfg->name);
	1355	+ return -EINVAL;
	1356	+ }
	1357	+ return 0;
	1358	+}
	1359	+
	1360	+/* Test one hash test vector in one configuration, using the ahash API */
	1361	+static int test_ahash_vec_cfg(const char *driver,
	1362	+ const struct hash_testvec *vec,
	1363	+ const char *vec_name,
	1364	+ const struct testvec_config *cfg,
	1365	+ struct ahash_request *req,
	1366	+ struct test_sglist *tsgl,
	1367	+ u8 *hashstate)
	1368	+{
	1369	+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
	1370	+ const unsigned int alignmask = crypto_ahash_alignmask(tfm);
	1371	+ const unsigned int digestsize = crypto_ahash_digestsize(tfm);
	1372	+ const unsigned int statesize = crypto_ahash_statesize(tfm);
	1373	+ const u32 req_flags = CRYPTO_TFM_REQ_MAY_BACKLOG \| cfg->req_flags;
	1374	+ const struct test_sg_division *divs[XBUFSIZE];
	1375	+ DECLARE_CRYPTO_WAIT(wait);
	1376	+ unsigned int i;
	1377	+ struct scatterlist *pending_sgl;
	1378	+ unsigned int pending_len;
	1379	+ u8 result[HASH_MAX_DIGESTSIZE + TESTMGR_POISON_LEN];
	1380	+ int err;
	1381	+
	1382	+ /* Set the key, if specified */
	1383	+ if (vec->ksize) {
	1384	+ err = do_setkey(crypto_ahash_setkey, tfm, vec->key, vec->ksize,
	1385	+ cfg, alignmask);
	1386	+ if (err) {
	1387	+ if (err == vec->setkey_error)
	1388	+ return 0;
	1389	+ pr_err("alg: ahash: %s setkey failed on test vector %s; expected_error=%d, actual_error=%d, flags=%#x\n",
	1390	+ driver, vec_name, vec->setkey_error, err,
	1391	+ crypto_ahash_get_flags(tfm));
	1392	+ return err;
	1393	+ }
	1394	+ if (vec->setkey_error) {
	1395	+ pr_err("alg: ahash: %s setkey unexpectedly succeeded on test vector %s; expected_error=%d\n",
	1396	+ driver, vec_name, vec->setkey_error);
	1397	+ return -EINVAL;
	1398	+ }
	1399	+ }
	1400	+
	1401	+ /* Build the scatterlist for the source data */
	1402	+ err = build_hash_sglist(tsgl, vec, cfg, alignmask, divs);
	1403	+ if (err) {
	1404	+ pr_err("alg: ahash: %s: error preparing scatterlist for test vector %s, cfg=\"%s\"\n",
	1405	+ driver, vec_name, cfg->name);
	1406	+ return err;
	1407	+ }
	1408	+
	1409	+ /* Do the actual hashing */
	1410	+
	1411	+ testmgr_poison(req->__ctx, crypto_ahash_reqsize(tfm));
	1412	+ testmgr_poison(result, digestsize + TESTMGR_POISON_LEN);
	1413	+
	1414	+ if (cfg->finalization_type == FINALIZATION_TYPE_DIGEST \|\|
	1415	+ vec->digest_error) {
	1416	+ /* Just using digest() */
	1417	+ ahash_request_set_callback(req, req_flags, crypto_req_done,
	1418	+ &wait);
	1419	+ ahash_request_set_crypt(req, tsgl->sgl, result, vec->psize);
	1420	+ err = do_ahash_op(crypto_ahash_digest, req, &wait, cfg->nosimd);
	1421	+ if (err) {
	1422	+ if (err == vec->digest_error)
	1423	+ return 0;
	1424	+ pr_err("alg: ahash: %s digest() failed on test vector %s; expected_error=%d, actual_error=%d, cfg=\"%s\"\n",
	1425	+ driver, vec_name, vec->digest_error, err,
	1426	+ cfg->name);
	1427	+ return err;
	1428	+ }
	1429	+ if (vec->digest_error) {
	1430	+ pr_err("alg: ahash: %s digest() unexpectedly succeeded on test vector %s; expected_error=%d, cfg=\"%s\"\n",
	1431	+ driver, vec_name, vec->digest_error, cfg->name);
	1432	+ return -EINVAL;
	1433	+ }
	1434	+ goto result_ready;
	1435	+ }
	1436	+
	1437	+ /* Using init(), zero or more update(), then final() or finup() */
	1438	+
	1439	+ ahash_request_set_callback(req, req_flags, crypto_req_done, &wait);
	1440	+ ahash_request_set_crypt(req, NULL, result, 0);
	1441	+ err = do_ahash_op(crypto_ahash_init, req, &wait, cfg->nosimd);
	1442	+ err = check_nonfinal_ahash_op("init", err, result, digestsize,
	1443	+ driver, vec_name, cfg);
	1444	+ if (err)
	1445	+ return err;
	1446	+
	1447	+ pending_sgl = NULL;
	1448	+ pending_len = 0;
	1449	+ for (i = 0; i < tsgl->nents; i++) {
	1450	+ if (divs[i]->flush_type != FLUSH_TYPE_NONE &&
	1451	+ pending_sgl != NULL) {
	1452	+ /* update() with the pending data */
	1453	+ ahash_request_set_callback(req, req_flags,
	1454	+ crypto_req_done, &wait);
	1455	+ ahash_request_set_crypt(req, pending_sgl, result,
	1456	+ pending_len);
	1457	+ err = do_ahash_op(crypto_ahash_update, req, &wait,
	1458	+ divs[i]->nosimd);
	1459	+ err = check_nonfinal_ahash_op("update", err,
	1460	+ result, digestsize,
	1461	+ driver, vec_name, cfg);
	1462	+ if (err)
	1463	+ return err;
	1464	+ pending_sgl = NULL;
	1465	+ pending_len = 0;
	1466	+ }
	1467	+ if (divs[i]->flush_type == FLUSH_TYPE_REIMPORT) {
	1468	+ /* Test ->export() and ->import() */
	1469	+ testmgr_poison(hashstate + statesize,
	1470	+ TESTMGR_POISON_LEN);
	1471	+ err = crypto_ahash_export(req, hashstate);
	1472	+ err = check_nonfinal_ahash_op("export", err,
	1473	+ result, digestsize,
	1474	+ driver, vec_name, cfg);
	1475	+ if (err)
	1476	+ return err;
	1477	+ if (!testmgr_is_poison(hashstate + statesize,
	1478	+ TESTMGR_POISON_LEN)) {
	1479	+ pr_err("alg: ahash: %s export() overran state buffer on test vector %s, cfg=\"%s\"\n",
	1480	+ driver, vec_name, cfg->name);
	1481	+ return -EOVERFLOW;
	1482	+ }
	1483	+
	1484	+ testmgr_poison(req->__ctx, crypto_ahash_reqsize(tfm));
	1485	+ err = crypto_ahash_import(req, hashstate);
	1486	+ err = check_nonfinal_ahash_op("import", err,
	1487	+ result, digestsize,
	1488	+ driver, vec_name, cfg);
	1489	+ if (err)
	1490	+ return err;
	1491	+ }
	1492	+ if (pending_sgl == NULL)
	1493	+ pending_sgl = &tsgl->sgl[i];
	1494	+ pending_len += tsgl->sgl[i].length;
	1495	+ }
	1496	+
	1497	+ ahash_request_set_callback(req, req_flags, crypto_req_done, &wait);
	1498	+ ahash_request_set_crypt(req, pending_sgl, result, pending_len);
	1499	+ if (cfg->finalization_type == FINALIZATION_TYPE_FINAL) {
	1500	+ /* finish with update() and final() */
	1501	+ err = do_ahash_op(crypto_ahash_update, req, &wait, cfg->nosimd);
	1502	+ err = check_nonfinal_ahash_op("update", err, result, digestsize,
	1503	+ driver, vec_name, cfg);
	1504	+ if (err)
	1505	+ return err;
	1506	+ err = do_ahash_op(crypto_ahash_final, req, &wait, cfg->nosimd);
	1507	+ if (err) {
	1508	+ pr_err("alg: ahash: %s final() failed with err %d on test vector %s, cfg=\"%s\"\n",
	1509	+ driver, err, vec_name, cfg->name);
	1510	+ return err;
	1511	+ }
	1512	+ } else {
	1513	+ /* finish with finup() */
	1514	+ err = do_ahash_op(crypto_ahash_finup, req, &wait, cfg->nosimd);
	1515	+ if (err) {
	1516	+ pr_err("alg: ahash: %s finup() failed with err %d on test vector %s, cfg=\"%s\"\n",
	1517	+ driver, err, vec_name, cfg->name);
	1518	+ return err;
	1519	+ }
	1520	+ }
	1521	+
	1522	+result_ready:
	1523	+ return check_hash_result("ahash", result, digestsize, vec, vec_name,
	1524	+ driver, cfg);
	1525	+}
	1526	+
	1527	+static int test_hash_vec_cfg(const char *driver,
	1528	+ const struct hash_testvec *vec,
	1529	+ const char *vec_name,
	1530	+ const struct testvec_config *cfg,
	1531	+ struct ahash_request *req,
	1532	+ struct shash_desc *desc,
	1533	+ struct test_sglist *tsgl,
	1534	+ u8 *hashstate)
	1535	+{
	1536	+ int err;
	1537	+
	1538	+ /*
	1539	+ * For algorithms implemented as "shash", most bugs will be detected by
	1540	+ * both the shash and ahash tests. Test the shash API first so that the
	1541	+ * failures involve less indirection, so are easier to debug.
	1542	+ */
	1543	+
	1544	+ if (desc) {
	1545	+ err = test_shash_vec_cfg(driver, vec, vec_name, cfg, desc, tsgl,
	1546	+ hashstate);
	1547	+ if (err)
	1548	+ return err;
	1549	+ }
	1550	+
	1551	+ return test_ahash_vec_cfg(driver, vec, vec_name, cfg, req, tsgl,
	1552	+ hashstate);
	1553	+}
	1554	+
	1555	+static int test_hash_vec(const char driver, const struct hash_testvec vec,
	1556	+ unsigned int vec_num, struct ahash_request *req,
	1557	+ struct shash_desc desc, struct test_sglist tsgl,
	1558	+ u8 *hashstate)
	1559	+{
	1560	+ char vec_name[16];
	1561	+ unsigned int i;
	1562	+ int err;
	1563	+
	1564	+ sprintf(vec_name, "%u", vec_num);
	1565	+
	1566	+ for (i = 0; i < ARRAY_SIZE(default_hash_testvec_configs); i++) {
	1567	+ err = test_hash_vec_cfg(driver, vec, vec_name,
	1568	+ &default_hash_testvec_configs[i],
	1569	+ req, desc, tsgl, hashstate);
	1570	+ if (err)
	1571	+ return err;
	1572	+ }
	1573	+
	1574	+#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS
	1575	+ if (!noextratests) {
	1576	+ struct testvec_config cfg;
	1577	+ char cfgname[TESTVEC_CONFIG_NAMELEN];
	1578	+
	1579	+ for (i = 0; i < fuzz_iterations; i++) {
	1580	+ generate_random_testvec_config(&cfg, cfgname,
	1581	+ sizeof(cfgname));
	1582	+ err = test_hash_vec_cfg(driver, vec, vec_name, &cfg,
	1583	+ req, desc, tsgl, hashstate);
	1584	+ if (err)
	1585	+ return err;
	1586	+ cond_resched();
	1587	+ }
	1588	+ }
	1589	+#endif
	1590	+ return 0;
	1591	+}
	1592	+
	1593	+#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS
	1594	+/*
	1595	+ * Generate a hash test vector from the given implementation.
	1596	+ * Assumes the buffers in 'vec' were already allocated.
	1597	+ */
	1598	+static void generate_random_hash_testvec(struct shash_desc *desc,
	1599	+ struct hash_testvec *vec,
	1600	+ unsigned int maxkeysize,
	1601	+ unsigned int maxdatasize,
	1602	+ char *name, size_t max_namelen)
	1603	+{
	1604	+ /* Data */
	1605	+ vec->psize = generate_random_length(maxdatasize);
	1606	+ generate_random_bytes((u8 *)vec->plaintext, vec->psize);
	1607	+
	1608	+ /*
	1609	+ * Key: length in range [1, maxkeysize], but usually choose maxkeysize.
	1610	+ * If algorithm is unkeyed, then maxkeysize == 0 and set ksize = 0.
	1611	+ */
	1612	+ vec->setkey_error = 0;
	1613	+ vec->ksize = 0;
	1614	+ if (maxkeysize) {
	1615	+ vec->ksize = maxkeysize;
	1616	+ if (prandom_u32() % 4 == 0)
	1617	+ vec->ksize = 1 + (prandom_u32() % maxkeysize);
	1618	+ generate_random_bytes((u8 *)vec->key, vec->ksize);
	1619	+
	1620	+ vec->setkey_error = crypto_shash_setkey(desc->tfm, vec->key,
	1621	+ vec->ksize);
	1622	+ /* If the key couldn't be set, no need to continue to digest. */
	1623	+ if (vec->setkey_error)
	1624	+ goto done;
	1625	+ }
	1626	+
	1627	+ /* Digest */
	1628	+ vec->digest_error = crypto_shash_digest(desc, vec->plaintext,
	1629	+ vec->psize, (u8 *)vec->digest);
	1630	+done:
	1631	+ snprintf(name, max_namelen, "\"random: psize=%u ksize=%u\"",
	1632	+ vec->psize, vec->ksize);
	1633	+}
	1634	+
	1635	+/*
	1636	+ * Test the hash algorithm represented by @req against the corresponding generic
	1637	+ * implementation, if one is available.
	1638	+ */
	1639	+static int test_hash_vs_generic_impl(const char *driver,
	1640	+ const char *generic_driver,
	1641	+ unsigned int maxkeysize,
	1642	+ struct ahash_request *req,
	1643	+ struct shash_desc *desc,
	1644	+ struct test_sglist *tsgl,
	1645	+ u8 *hashstate)
	1646	+{
	1647	+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
	1648	+ const unsigned int digestsize = crypto_ahash_digestsize(tfm);
	1649	+ const unsigned int blocksize = crypto_ahash_blocksize(tfm);
	1650	+ const unsigned int maxdatasize = (2 * PAGE_SIZE) - TESTMGR_POISON_LEN;
	1651	+ const char *algname = crypto_hash_alg_common(tfm)->base.cra_name;
	1652	+ char _generic_driver[CRYPTO_MAX_ALG_NAME];
	1653	+ struct crypto_shash *generic_tfm = NULL;
	1654	+ struct shash_desc *generic_desc = NULL;
	1655	+ unsigned int i;
	1656	+ struct hash_testvec vec = { 0 };
	1657	+ char vec_name[64];
	1658	+ struct testvec_config *cfg;
	1659	+ char cfgname[TESTVEC_CONFIG_NAMELEN];
	1660	+ int err;
	1661	+
	1662	+ if (noextratests)
	1663	+ return 0;
	1664	+
	1665	+ if (!generic_driver) { /* Use default naming convention? */
	1666	+ err = build_generic_driver_name(algname, _generic_driver);
	1667	+ if (err)
	1668	+ return err;
	1669	+ generic_driver = _generic_driver;
	1670	+ }
	1671	+
	1672	+ if (strcmp(generic_driver, driver) == 0) /* Already the generic impl? */
	1673	+ return 0;
	1674	+
	1675	+ generic_tfm = crypto_alloc_shash(generic_driver, 0, 0);
	1676	+ if (IS_ERR(generic_tfm)) {
	1677	+ err = PTR_ERR(generic_tfm);
	1678	+ if (err == -ENOENT) {
	1679	+ pr_warn("alg: hash: skipping comparison tests for %s because %s is unavailable\n",
	1680	+ driver, generic_driver);
	1681	+ return 0;
	1682	+ }
	1683	+ pr_err("alg: hash: error allocating %s (generic impl of %s): %d\n",
	1684	+ generic_driver, algname, err);
	1685	+ return err;
	1686	+ }
	1687	+
	1688	+ cfg = kzalloc(sizeof(*cfg), GFP_KERNEL);
	1689	+ if (!cfg) {
	1690	+ err = -ENOMEM;
	1691	+ goto out;
	1692	+ }
	1693	+
	1694	+ generic_desc = kzalloc(sizeof(*desc) +
	1695	+ crypto_shash_descsize(generic_tfm), GFP_KERNEL);
	1696	+ if (!generic_desc) {
	1697	+ err = -ENOMEM;
	1698	+ goto out;
	1699	+ }
	1700	+ generic_desc->tfm = generic_tfm;
	1701	+
	1702	+ /* Check the algorithm properties for consistency. */
	1703	+
	1704	+ if (digestsize != crypto_shash_digestsize(generic_tfm)) {
	1705	+ pr_err("alg: hash: digestsize for %s (%u) doesn't match generic impl (%u)\n",
	1706	+ driver, digestsize,
	1707	+ crypto_shash_digestsize(generic_tfm));
	1708	+ err = -EINVAL;
	1709	+ goto out;
	1710	+ }
	1711	+
	1712	+ if (blocksize != crypto_shash_blocksize(generic_tfm)) {
	1713	+ pr_err("alg: hash: blocksize for %s (%u) doesn't match generic impl (%u)\n",
	1714	+ driver, blocksize, crypto_shash_blocksize(generic_tfm));
	1715	+ err = -EINVAL;
	1716	+ goto out;
	1717	+ }
	1718	+
	1719	+ /*
	1720	+ * Now generate test vectors using the generic implementation, and test
	1721	+ * the other implementation against them.
	1722	+ */
	1723	+
	1724	+ vec.key = kmalloc(maxkeysize, GFP_KERNEL);
	1725	+ vec.plaintext = kmalloc(maxdatasize, GFP_KERNEL);
	1726	+ vec.digest = kmalloc(digestsize, GFP_KERNEL);
	1727	+ if (!vec.key \|\| !vec.plaintext \|\| !vec.digest) {
	1728	+ err = -ENOMEM;
	1729	+ goto out;
	1730	+ }
	1731	+
	1732	+ for (i = 0; i < fuzz_iterations * 8; i++) {
	1733	+ generate_random_hash_testvec(generic_desc, &vec,
	1734	+ maxkeysize, maxdatasize,
	1735	+ vec_name, sizeof(vec_name));
	1736	+ generate_random_testvec_config(cfg, cfgname, sizeof(cfgname));
	1737	+
	1738	+ err = test_hash_vec_cfg(driver, &vec, vec_name, cfg,
	1739	+ req, desc, tsgl, hashstate);
	1740	+ if (err)
	1741	+ goto out;
	1742	+ cond_resched();
	1743	+ }
	1744	+ err = 0;
	1745	+out:
	1746	+ kfree(cfg);
	1747	+ kfree(vec.key);
	1748	+ kfree(vec.plaintext);
	1749	+ kfree(vec.digest);
	1750	+ crypto_free_shash(generic_tfm);
	1751	+ kfree_sensitive(generic_desc);
	1752	+ return err;
	1753	+}
	1754	+#else /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */
	1755	+static int test_hash_vs_generic_impl(const char *driver,
	1756	+ const char *generic_driver,
	1757	+ unsigned int maxkeysize,
	1758	+ struct ahash_request *req,
	1759	+ struct shash_desc *desc,
	1760	+ struct test_sglist *tsgl,
	1761	+ u8 *hashstate)
	1762	+{
	1763	+ return 0;
	1764	+}
	1765	+#endif /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */
	1766	+
	1767	+static int alloc_shash(const char *driver, u32 type, u32 mask,
	1768	+ struct crypto_shash **tfm_ret,
	1769	+ struct shash_desc **desc_ret)
	1770	+{
	1771	+ struct crypto_shash *tfm;
	1772	+ struct shash_desc *desc;
	1773	+
	1774	+ tfm = crypto_alloc_shash(driver, type, mask);
	1775	+ if (IS_ERR(tfm)) {
	1776	+ if (PTR_ERR(tfm) == -ENOENT) {
	1777	+ /*
	1778	+ * This algorithm is only available through the ahash
	1779	+ * API, not the shash API, so skip the shash tests.
	1780	+ */
	1781	+ return 0;
	1782	+ }
	1783	+ pr_err("alg: hash: failed to allocate shash transform for %s: %ld\n",
	1784	+ driver, PTR_ERR(tfm));
	1785	+ return PTR_ERR(tfm);
	1786	+ }
	1787	+
	1788	+ desc = kmalloc(sizeof(*desc) + crypto_shash_descsize(tfm), GFP_KERNEL);
	1789	+ if (!desc) {
	1790	+ crypto_free_shash(tfm);
	1791	+ return -ENOMEM;
	1792	+ }
	1793	+ desc->tfm = tfm;
	1794	+
	1795	+ *tfm_ret = tfm;
	1796	+ *desc_ret = desc;
	1797	+ return 0;
	1798	+}
	1799	+
	1800	+static int __alg_test_hash(const struct hash_testvec *vecs,
	1801	+ unsigned int num_vecs, const char *driver,
	1802	+ u32 type, u32 mask,
	1803	+ const char *generic_driver, unsigned int maxkeysize)
	1804	+{
	1805	+ struct crypto_ahash *atfm = NULL;
	1806	+ struct ahash_request *req = NULL;
	1807	+ struct crypto_shash *stfm = NULL;
	1808	+ struct shash_desc *desc = NULL;
	1809	+ struct test_sglist *tsgl = NULL;
	1810	+ u8 *hashstate = NULL;
	1811	+ unsigned int statesize;
	1812	+ unsigned int i;
	1813	+ int err;
	1814	+
	1815	+ /*
	1816	+ * Always test the ahash API. This works regardless of whether the
	1817	+ * algorithm is implemented as ahash or shash.
	1818	+ */
	1819	+
	1820	+ atfm = crypto_alloc_ahash(driver, type, mask);
	1821	+ if (IS_ERR(atfm)) {
	1822	+ pr_err("alg: hash: failed to allocate transform for %s: %ld\n",
	1823	+ driver, PTR_ERR(atfm));
	1824	+ return PTR_ERR(atfm);
	1825	+ }
	1826	+
	1827	+ req = ahash_request_alloc(atfm, GFP_KERNEL);
	1828	+ if (!req) {
	1829	+ pr_err("alg: hash: failed to allocate request for %s\n",
	1830	+ driver);
	1831	+ err = -ENOMEM;
	1832	+ goto out;
	1833	+ }
	1834	+
	1835	+ /*
	1836	+ * If available also test the shash API, to cover corner cases that may
	1837	+ * be missed by testing the ahash API only.
	1838	+ */
	1839	+ err = alloc_shash(driver, type, mask, &stfm, &desc);
	1840	+ if (err)
	1841	+ goto out;
	1842	+
	1843	+ tsgl = kmalloc(sizeof(*tsgl), GFP_KERNEL);
	1844	+ if (!tsgl \|\| init_test_sglist(tsgl) != 0) {
	1845	+ pr_err("alg: hash: failed to allocate test buffers for %s\n",
	1846	+ driver);
	1847	+ kfree(tsgl);
	1848	+ tsgl = NULL;
	1849	+ err = -ENOMEM;
	1850	+ goto out;
	1851	+ }
	1852	+
	1853	+ statesize = crypto_ahash_statesize(atfm);
	1854	+ if (stfm)
	1855	+ statesize = max(statesize, crypto_shash_statesize(stfm));
	1856	+ hashstate = kmalloc(statesize + TESTMGR_POISON_LEN, GFP_KERNEL);
	1857	+ if (!hashstate) {
	1858	+ pr_err("alg: hash: failed to allocate hash state buffer for %s\n",
	1859	+ driver);
	1860	+ err = -ENOMEM;
	1861	+ goto out;
	1862	+ }
	1863	+
	1864	+ for (i = 0; i < num_vecs; i++) {
	1865	+ err = test_hash_vec(driver, &vecs[i], i, req, desc, tsgl,
	1866	+ hashstate);
	1867	+ if (err)
	1868	+ goto out;
	1869	+ cond_resched();
	1870	+ }
	1871	+ err = test_hash_vs_generic_impl(driver, generic_driver, maxkeysize, req,
	1872	+ desc, tsgl, hashstate);
	1873	+out:
	1874	+ kfree(hashstate);
	1875	+ if (tsgl) {
	1876	+ destroy_test_sglist(tsgl);
	1877	+ kfree(tsgl);
	1878	+ }
	1879	+ kfree(desc);
	1880	+ crypto_free_shash(stfm);
	1881	+ ahash_request_free(req);
	1882	+ crypto_free_ahash(atfm);
	1883	+ return err;
	1884	+}
	1885	+
	1886	+static int alg_test_hash(const struct alg_test_desc desc, const char driver,
	1887	+ u32 type, u32 mask)
	1888	+{
	1889	+ const struct hash_testvec *template = desc->suite.hash.vecs;
	1890	+ unsigned int tcount = desc->suite.hash.count;
	1891	+ unsigned int nr_unkeyed, nr_keyed;
	1892	+ unsigned int maxkeysize = 0;
	1893	+ int err;
	1894	+
	1895	+ /*
	1896	+ * For OPTIONAL_KEY algorithms, we have to do all the unkeyed tests
	1897	+ * first, before setting a key on the tfm. To make this easier, we
	1898	+ * require that the unkeyed test vectors (if any) are listed first.
	1899	+ */
	1900	+
	1901	+ for (nr_unkeyed = 0; nr_unkeyed < tcount; nr_unkeyed++) {
	1902	+ if (template[nr_unkeyed].ksize)
	1903	+ break;
	1904	+ }
	1905	+ for (nr_keyed = 0; nr_unkeyed + nr_keyed < tcount; nr_keyed++) {
	1906	+ if (!template[nr_unkeyed + nr_keyed].ksize) {
	1907	+ pr_err("alg: hash: test vectors for %s out of order, "
	1908	+ "unkeyed ones must come first\n", desc->alg);
	1909	+ return -EINVAL;
	1910	+ }
	1911	+ maxkeysize = max_t(unsigned int, maxkeysize,
	1912	+ template[nr_unkeyed + nr_keyed].ksize);
	1913	+ }
	1914	+
	1915	+ err = 0;
	1916	+ if (nr_unkeyed) {
	1917	+ err = __alg_test_hash(template, nr_unkeyed, driver, type, mask,
	1918	+ desc->generic_driver, maxkeysize);
	1919	+ template += nr_unkeyed;
	1920	+ }
	1921	+
	1922	+ if (!err && nr_keyed)
	1923	+ err = __alg_test_hash(template, nr_keyed, driver, type, mask,
	1924	+ desc->generic_driver, maxkeysize);
	1925	+
	1926	+ return err;
	1927	+}
	1928	+
	1929	+static int test_aead_vec_cfg(const char *driver, int enc,
	1930	+ const struct aead_testvec *vec,
	1931	+ const char *vec_name,
	1932	+ const struct testvec_config *cfg,
	1933	+ struct aead_request *req,
	1934	+ struct cipher_test_sglists *tsgls)
	1935	+{
	1936	+ struct crypto_aead *tfm = crypto_aead_reqtfm(req);
	1937	+ const unsigned int alignmask = crypto_aead_alignmask(tfm);
	1938	+ const unsigned int ivsize = crypto_aead_ivsize(tfm);
	1939	+ const unsigned int authsize = vec->clen - vec->plen;
	1940	+ const u32 req_flags = CRYPTO_TFM_REQ_MAY_BACKLOG \| cfg->req_flags;
	1941	+ const char *op = enc ? "encryption" : "decryption";
	1942	+ DECLARE_CRYPTO_WAIT(wait);
	1943	+ u8 _iv[3 * (MAX_ALGAPI_ALIGNMASK + 1) + MAX_IVLEN];
	1944	+ u8 iv = PTR_ALIGN(&_iv[0], 2 (MAX_ALGAPI_ALIGNMASK + 1)) +
	1945	+ cfg->iv_offset +
	1946	+ (cfg->iv_offset_relative_to_alignmask ? alignmask : 0);
	1947	+ struct kvec input[2];
	1948	+ int err;
	1949	+
	1950	+ /* Set the key */
	1951	+ if (vec->wk)
	1952	+ crypto_aead_set_flags(tfm, CRYPTO_TFM_REQ_FORBID_WEAK_KEYS);
	1953	+ else
	1954	+ crypto_aead_clear_flags(tfm, CRYPTO_TFM_REQ_FORBID_WEAK_KEYS);
	1955	+
	1956	+ err = do_setkey(crypto_aead_setkey, tfm, vec->key, vec->klen,
	1957	+ cfg, alignmask);
	1958	+ if (err && err != vec->setkey_error) {
	1959	+ pr_err("alg: aead: %s setkey failed on test vector %s; expected_error=%d, actual_error=%d, flags=%#x\n",
	1960	+ driver, vec_name, vec->setkey_error, err,
	1961	+ crypto_aead_get_flags(tfm));
	1962	+ return err;
	1963	+ }
	1964	+ if (!err && vec->setkey_error) {
	1965	+ pr_err("alg: aead: %s setkey unexpectedly succeeded on test vector %s; expected_error=%d\n",
	1966	+ driver, vec_name, vec->setkey_error);
	1967	+ return -EINVAL;
	1968	+ }
	1969	+
	1970	+ /* Set the authentication tag size */
	1971	+ err = crypto_aead_setauthsize(tfm, authsize);
	1972	+ if (err && err != vec->setauthsize_error) {
	1973	+ pr_err("alg: aead: %s setauthsize failed on test vector %s; expected_error=%d, actual_error=%d\n",
	1974	+ driver, vec_name, vec->setauthsize_error, err);
	1975	+ return err;
	1976	+ }
	1977	+ if (!err && vec->setauthsize_error) {
	1978	+ pr_err("alg: aead: %s setauthsize unexpectedly succeeded on test vector %s; expected_error=%d\n",
	1979	+ driver, vec_name, vec->setauthsize_error);
	1980	+ return -EINVAL;
	1981	+ }
	1982	+
	1983	+ if (vec->setkey_error \|\| vec->setauthsize_error)
	1984	+ return 0;
	1985	+
	1986	+ /* The IV must be copied to a buffer, as the algorithm may modify it */
	1987	+ if (WARN_ON(ivsize > MAX_IVLEN))
	1988	+ return -EINVAL;
	1989	+ if (vec->iv)
	1990	+ memcpy(iv, vec->iv, ivsize);
	1991	+ else
	1992	+ memset(iv, 0, ivsize);
	1993	+
	1994	+ /* Build the src/dst scatterlists */
	1995	+ input[0].iov_base = (void *)vec->assoc;
	1996	+ input[0].iov_len = vec->alen;
	1997	+ input[1].iov_base = enc ? (void )vec->ptext : (void )vec->ctext;
	1998	+ input[1].iov_len = enc ? vec->plen : vec->clen;
	1999	+ err = build_cipher_test_sglists(tsgls, cfg, alignmask,
	2000	+ vec->alen + (enc ? vec->plen :
	2001	+ vec->clen),
	2002	+ vec->alen + (enc ? vec->clen :
	2003	+ vec->plen),
	2004	+ input, 2);
	2005	+ if (err) {
	2006	+ pr_err("alg: aead: %s %s: error preparing scatterlists for test vector %s, cfg=\"%s\"\n",
	2007	+ driver, op, vec_name, cfg->name);
	2008	+ return err;
	2009	+ }
	2010	+
	2011	+ /* Do the actual encryption or decryption */
	2012	+ testmgr_poison(req->__ctx, crypto_aead_reqsize(tfm));
	2013	+ aead_request_set_callback(req, req_flags, crypto_req_done, &wait);
	2014	+ aead_request_set_crypt(req, tsgls->src.sgl_ptr, tsgls->dst.sgl_ptr,
	2015	+ enc ? vec->plen : vec->clen, iv);
	2016	+ aead_request_set_ad(req, vec->alen);
	2017	+ if (cfg->nosimd)
	2018	+ crypto_disable_simd_for_test();
	2019	+ err = enc ? crypto_aead_encrypt(req) : crypto_aead_decrypt(req);
	2020	+ if (cfg->nosimd)
	2021	+ crypto_reenable_simd_for_test();
	2022	+ err = crypto_wait_req(err, &wait);
	2023	+
	2024	+ /* Check that the algorithm didn't overwrite things it shouldn't have */
	2025	+ if (req->cryptlen != (enc ? vec->plen : vec->clen) \|\|
	2026	+ req->assoclen != vec->alen \|\|
	2027	+ req->iv != iv \|\|
	2028	+ req->src != tsgls->src.sgl_ptr \|\|
	2029	+ req->dst != tsgls->dst.sgl_ptr \|\|
	2030	+ crypto_aead_reqtfm(req) != tfm \|\|
	2031	+ req->base.complete != crypto_req_done \|\|
	2032	+ req->base.flags != req_flags \|\|
	2033	+ req->base.data != &wait) {
	2034	+ pr_err("alg: aead: %s %s corrupted request struct on test vector %s, cfg=\"%s\"\n",
	2035	+ driver, op, vec_name, cfg->name);
	2036	+ if (req->cryptlen != (enc ? vec->plen : vec->clen))
	2037	+ pr_err("alg: aead: changed 'req->cryptlen'\n");
	2038	+ if (req->assoclen != vec->alen)
	2039	+ pr_err("alg: aead: changed 'req->assoclen'\n");
	2040	+ if (req->iv != iv)
	2041	+ pr_err("alg: aead: changed 'req->iv'\n");
	2042	+ if (req->src != tsgls->src.sgl_ptr)
	2043	+ pr_err("alg: aead: changed 'req->src'\n");
	2044	+ if (req->dst != tsgls->dst.sgl_ptr)
	2045	+ pr_err("alg: aead: changed 'req->dst'\n");
	2046	+ if (crypto_aead_reqtfm(req) != tfm)
	2047	+ pr_err("alg: aead: changed 'req->base.tfm'\n");
	2048	+ if (req->base.complete != crypto_req_done)
	2049	+ pr_err("alg: aead: changed 'req->base.complete'\n");
	2050	+ if (req->base.flags != req_flags)
	2051	+ pr_err("alg: aead: changed 'req->base.flags'\n");
	2052	+ if (req->base.data != &wait)
	2053	+ pr_err("alg: aead: changed 'req->base.data'\n");
	2054	+ return -EINVAL;
	2055	+ }
	2056	+ if (is_test_sglist_corrupted(&tsgls->src)) {
	2057	+ pr_err("alg: aead: %s %s corrupted src sgl on test vector %s, cfg=\"%s\"\n",
	2058	+ driver, op, vec_name, cfg->name);
	2059	+ return -EINVAL;
	2060	+ }
	2061	+ if (tsgls->dst.sgl_ptr != tsgls->src.sgl &&
	2062	+ is_test_sglist_corrupted(&tsgls->dst)) {
	2063	+ pr_err("alg: aead: %s %s corrupted dst sgl on test vector %s, cfg=\"%s\"\n",
	2064	+ driver, op, vec_name, cfg->name);
	2065	+ return -EINVAL;
	2066	+ }
	2067	+
	2068	+ /* Check for unexpected success or failure, or wrong error code */
	2069	+ if ((err == 0 && vec->novrfy) \|\|
	2070	+ (err != vec->crypt_error && !(err == -EBADMSG && vec->novrfy))) {
	2071	+ char expected_error[32];
	2072	+
	2073	+ if (vec->novrfy &&
	2074	+ vec->crypt_error != 0 && vec->crypt_error != -EBADMSG)
	2075	+ sprintf(expected_error, "-EBADMSG or %d",
	2076	+ vec->crypt_error);
	2077	+ else if (vec->novrfy)
	2078	+ sprintf(expected_error, "-EBADMSG");
	2079	+ else
	2080	+ sprintf(expected_error, "%d", vec->crypt_error);
	2081	+ if (err) {
	2082	+ pr_err("alg: aead: %s %s failed on test vector %s; expected_error=%s, actual_error=%d, cfg=\"%s\"\n",
	2083	+ driver, op, vec_name, expected_error, err,
	2084	+ cfg->name);
	2085	+ return err;
	2086	+ }
	2087	+ pr_err("alg: aead: %s %s unexpectedly succeeded on test vector %s; expected_error=%s, cfg=\"%s\"\n",
	2088	+ driver, op, vec_name, expected_error, cfg->name);
	2089	+ return -EINVAL;
	2090	+ }
	2091	+ if (err) /* Expectedly failed. */
	2092	+ return 0;
	2093	+
	2094	+ /* Check for the correct output (ciphertext or plaintext) */
	2095	+ err = verify_correct_output(&tsgls->dst, enc ? vec->ctext : vec->ptext,
	2096	+ enc ? vec->clen : vec->plen,
	2097	+ vec->alen, enc \|\| !cfg->inplace);
	2098	+ if (err == -EOVERFLOW) {
	2099	+ pr_err("alg: aead: %s %s overran dst buffer on test vector %s, cfg=\"%s\"\n",
	2100	+ driver, op, vec_name, cfg->name);
	2101	+ return err;
	2102	+ }
	2103	+ if (err) {
	2104	+ pr_err("alg: aead: %s %s test failed (wrong result) on test vector %s, cfg=\"%s\"\n",
	2105	+ driver, op, vec_name, cfg->name);
	2106	+ return err;
	2107	+ }
	2108	+
	2109	+ return 0;
	2110	+}
	2111	+
	2112	+static int test_aead_vec(const char *driver, int enc,
	2113	+ const struct aead_testvec *vec, unsigned int vec_num,
	2114	+ struct aead_request *req,
	2115	+ struct cipher_test_sglists *tsgls)
	2116	+{
	2117	+ char vec_name[16];
	2118	+ unsigned int i;
	2119	+ int err;
	2120	+
	2121	+ if (enc && vec->novrfy)
	2122	+ return 0;
	2123	+
	2124	+ sprintf(vec_name, "%u", vec_num);
	2125	+
	2126	+ for (i = 0; i < ARRAY_SIZE(default_cipher_testvec_configs); i++) {
	2127	+ err = test_aead_vec_cfg(driver, enc, vec, vec_name,
	2128	+ &default_cipher_testvec_configs[i],
	2129	+ req, tsgls);
	2130	+ if (err)
	2131	+ return err;
	2132	+ }
	2133	+
	2134	+#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS
	2135	+ if (!noextratests) {
	2136	+ struct testvec_config cfg;
	2137	+ char cfgname[TESTVEC_CONFIG_NAMELEN];
	2138	+
	2139	+ for (i = 0; i < fuzz_iterations; i++) {
	2140	+ generate_random_testvec_config(&cfg, cfgname,
	2141	+ sizeof(cfgname));
	2142	+ err = test_aead_vec_cfg(driver, enc, vec, vec_name,
	2143	+ &cfg, req, tsgls);
	2144	+ if (err)
	2145	+ return err;
	2146	+ cond_resched();
	2147	+ }
	2148	+ }
	2149	+#endif
	2150	+ return 0;
	2151	+}
	2152	+
	2153	+#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS
	2154	+
	2155	+struct aead_extra_tests_ctx {
	2156	+ struct aead_request *req;
	2157	+ struct crypto_aead *tfm;
	2158	+ const char *driver;
	2159	+ const struct alg_test_desc *test_desc;
	2160	+ struct cipher_test_sglists *tsgls;
	2161	+ unsigned int maxdatasize;
	2162	+ unsigned int maxkeysize;
	2163	+
	2164	+ struct aead_testvec vec;
	2165	+ char vec_name[64];
	2166	+ char cfgname[TESTVEC_CONFIG_NAMELEN];
	2167	+ struct testvec_config cfg;
	2168	+};
	2169	+
	2170	+/*
	2171	+ * Make at least one random change to a (ciphertext, AAD) pair. "Ciphertext"
	2172	+ * here means the full ciphertext including the authentication tag. The
	2173	+ * authentication tag (and hence also the ciphertext) is assumed to be nonempty.
	2174	+ */
	2175	+static void mutate_aead_message(struct aead_testvec *vec, bool aad_iv,
	2176	+ unsigned int ivsize)
	2177	+{
	2178	+ const unsigned int aad_tail_size = aad_iv ? ivsize : 0;
	2179	+ const unsigned int authsize = vec->clen - vec->plen;
	2180	+
	2181	+ if (prandom_u32() % 2 == 0 && vec->alen > aad_tail_size) {
	2182	+ /* Mutate the AAD */
	2183	+ flip_random_bit((u8 *)vec->assoc, vec->alen - aad_tail_size);
	2184	+ if (prandom_u32() % 2 == 0)
	2185	+ return;
	2186	+ }
	2187	+ if (prandom_u32() % 2 == 0) {
	2188	+ /* Mutate auth tag (assuming it's at the end of ciphertext) */
	2189	+ flip_random_bit((u8 *)vec->ctext + vec->plen, authsize);
	2190	+ } else {
	2191	+ /* Mutate any part of the ciphertext */
	2192	+ flip_random_bit((u8 *)vec->ctext, vec->clen);
	2193	+ }
	2194	+}
	2195	+
	2196	+/*
	2197	+ * Minimum authentication tag size in bytes at which we assume that we can
	2198	+ * reliably generate inauthentic messages, i.e. not generate an authentic
	2199	+ * message by chance.
	2200	+ */
	2201	+#define MIN_COLLISION_FREE_AUTHSIZE 8
	2202	+
	2203	+static void generate_aead_message(struct aead_request *req,
	2204	+ const struct aead_test_suite *suite,
	2205	+ struct aead_testvec *vec,
	2206	+ bool prefer_inauthentic)
	2207	+{
	2208	+ struct crypto_aead *tfm = crypto_aead_reqtfm(req);
	2209	+ const unsigned int ivsize = crypto_aead_ivsize(tfm);
	2210	+ const unsigned int authsize = vec->clen - vec->plen;
	2211	+ const bool inauthentic = (authsize >= MIN_COLLISION_FREE_AUTHSIZE) &&
	2212	+ (prefer_inauthentic \|\| prandom_u32() % 4 == 0);
	2213	+
	2214	+ /* Generate the AAD. */
	2215	+ generate_random_bytes((u8 *)vec->assoc, vec->alen);
	2216	+ if (suite->aad_iv && vec->alen >= ivsize)
	2217	+ /* Avoid implementation-defined behavior. */
	2218	+ memcpy((u8 *)vec->assoc + vec->alen - ivsize, vec->iv, ivsize);
	2219	+
	2220	+ if (inauthentic && prandom_u32() % 2 == 0) {
	2221	+ /* Generate a random ciphertext. */
	2222	+ generate_random_bytes((u8 *)vec->ctext, vec->clen);
	2223	+ } else {
	2224	+ int i = 0;
	2225	+ struct scatterlist src[2], dst;
	2226	+ u8 iv[MAX_IVLEN];
	2227	+ DECLARE_CRYPTO_WAIT(wait);
	2228	+
	2229	+ /* Generate a random plaintext and encrypt it. */
	2230	+ sg_init_table(src, 2);
	2231	+ if (vec->alen)
	2232	+ sg_set_buf(&src[i++], vec->assoc, vec->alen);
	2233	+ if (vec->plen) {
	2234	+ generate_random_bytes((u8 *)vec->ptext, vec->plen);
	2235	+ sg_set_buf(&src[i++], vec->ptext, vec->plen);
	2236	+ }
	2237	+ sg_init_one(&dst, vec->ctext, vec->alen + vec->clen);
	2238	+ memcpy(iv, vec->iv, ivsize);
	2239	+ aead_request_set_callback(req, 0, crypto_req_done, &wait);
	2240	+ aead_request_set_crypt(req, src, &dst, vec->plen, iv);
	2241	+ aead_request_set_ad(req, vec->alen);
	2242	+ vec->crypt_error = crypto_wait_req(crypto_aead_encrypt(req),
	2243	+ &wait);
	2244	+ /* If encryption failed, we're done. */
	2245	+ if (vec->crypt_error != 0)
	2246	+ return;
	2247	+ memmove((u8 *)vec->ctext, vec->ctext + vec->alen, vec->clen);
	2248	+ if (!inauthentic)
	2249	+ return;
	2250	+ /*
	2251	+ * Mutate the authentic (ciphertext, AAD) pair to get an
	2252	+ * inauthentic one.
	2253	+ */
	2254	+ mutate_aead_message(vec, suite->aad_iv, ivsize);
	2255	+ }
	2256	+ vec->novrfy = 1;
	2257	+ if (suite->einval_allowed)
	2258	+ vec->crypt_error = -EINVAL;
	2259	+}
	2260	+
	2261	+/*
	2262	+ * Generate an AEAD test vector 'vec' using the implementation specified by
	2263	+ * 'req'. The buffers in 'vec' must already be allocated.
	2264	+ *
	2265	+ * If 'prefer_inauthentic' is true, then this function will generate inauthentic
	2266	+ * test vectors (i.e. vectors with 'vec->novrfy=1') more often.
	2267	+ */
	2268	+static void generate_random_aead_testvec(struct aead_request *req,
	2269	+ struct aead_testvec *vec,
	2270	+ const struct aead_test_suite *suite,
	2271	+ unsigned int maxkeysize,
	2272	+ unsigned int maxdatasize,
	2273	+ char *name, size_t max_namelen,
	2274	+ bool prefer_inauthentic)
	2275	+{
	2276	+ struct crypto_aead *tfm = crypto_aead_reqtfm(req);
	2277	+ const unsigned int ivsize = crypto_aead_ivsize(tfm);
	2278	+ const unsigned int maxauthsize = crypto_aead_maxauthsize(tfm);
	2279	+ unsigned int authsize;
	2280	+ unsigned int total_len;
	2281	+
	2282	+ /* Key: length in [0, maxkeysize], but usually choose maxkeysize */
	2283	+ vec->klen = maxkeysize;
	2284	+ if (prandom_u32() % 4 == 0)
	2285	+ vec->klen = prandom_u32() % (maxkeysize + 1);
	2286	+ generate_random_bytes((u8 *)vec->key, vec->klen);
	2287	+ vec->setkey_error = crypto_aead_setkey(tfm, vec->key, vec->klen);
	2288	+
	2289	+ /* IV */
	2290	+ generate_random_bytes((u8 *)vec->iv, ivsize);
	2291	+
	2292	+ /* Tag length: in [0, maxauthsize], but usually choose maxauthsize */
	2293	+ authsize = maxauthsize;
	2294	+ if (prandom_u32() % 4 == 0)
	2295	+ authsize = prandom_u32() % (maxauthsize + 1);
	2296	+ if (prefer_inauthentic && authsize < MIN_COLLISION_FREE_AUTHSIZE)
	2297	+ authsize = MIN_COLLISION_FREE_AUTHSIZE;
	2298	+ if (WARN_ON(authsize > maxdatasize))
	2299	+ authsize = maxdatasize;
	2300	+ maxdatasize -= authsize;
	2301	+ vec->setauthsize_error = crypto_aead_setauthsize(tfm, authsize);
	2302	+
	2303	+ /* AAD, plaintext, and ciphertext lengths */
	2304	+ total_len = generate_random_length(maxdatasize);
	2305	+ if (prandom_u32() % 4 == 0)
	2306	+ vec->alen = 0;
	2307	+ else
	2308	+ vec->alen = generate_random_length(total_len);
	2309	+ vec->plen = total_len - vec->alen;
	2310	+ vec->clen = vec->plen + authsize;
	2311	+
	2312	+ /*
	2313	+ * Generate the AAD, plaintext, and ciphertext. Not applicable if the
	2314	+ * key or the authentication tag size couldn't be set.
	2315	+ */
	2316	+ vec->novrfy = 0;
	2317	+ vec->crypt_error = 0;
	2318	+ if (vec->setkey_error == 0 && vec->setauthsize_error == 0)
	2319	+ generate_aead_message(req, suite, vec, prefer_inauthentic);
	2320	+ snprintf(name, max_namelen,
	2321	+ "\"random: alen=%u plen=%u authsize=%u klen=%u novrfy=%d\"",
	2322	+ vec->alen, vec->plen, authsize, vec->klen, vec->novrfy);
	2323	+}
	2324	+
	2325	+static void try_to_generate_inauthentic_testvec(
	2326	+ struct aead_extra_tests_ctx *ctx)
179	2327	{
180	2328	int i;
181	2329
182		- for (i = 0; i < size; i++) {
183		- if (result[i] != c)
184		- return -EINVAL;
	2330	+ for (i = 0; i < 10; i++) {
	2331	+ generate_random_aead_testvec(ctx->req, &ctx->vec,
	2332	+ &ctx->test_desc->suite.aead,
	2333	+ ctx->maxkeysize, ctx->maxdatasize,
	2334	+ ctx->vec_name,
	2335	+ sizeof(ctx->vec_name), true);
	2336	+ if (ctx->vec.novrfy)
	2337	+ return;
185	2338	}
	2339	+}
186	2340
	2341	+/*
	2342	+ * Generate inauthentic test vectors (i.e. ciphertext, AAD pairs that aren't the
	2343	+ * result of an encryption with the key) and verify that decryption fails.
	2344	+ */
	2345	+static int test_aead_inauthentic_inputs(struct aead_extra_tests_ctx *ctx)
	2346	+{
	2347	+ unsigned int i;
	2348	+ int err;
	2349	+
	2350	+ for (i = 0; i < fuzz_iterations * 8; i++) {
	2351	+ /*
	2352	+ * Since this part of the tests isn't comparing the
	2353	+ * implementation to another, there's no point in testing any
	2354	+ * test vectors other than inauthentic ones (vec.novrfy=1) here.
	2355	+ *
	2356	+ * If we're having trouble generating such a test vector, e.g.
	2357	+ * if the algorithm keeps rejecting the generated keys, don't
	2358	+ * retry forever; just continue on.
	2359	+ */
	2360	+ try_to_generate_inauthentic_testvec(ctx);
	2361	+ if (ctx->vec.novrfy) {
	2362	+ generate_random_testvec_config(&ctx->cfg, ctx->cfgname,
	2363	+ sizeof(ctx->cfgname));
	2364	+ err = test_aead_vec_cfg(ctx->driver, DECRYPT, &ctx->vec,
	2365	+ ctx->vec_name, &ctx->cfg,
	2366	+ ctx->req, ctx->tsgls);
	2367	+ if (err)
	2368	+ return err;
	2369	+ }
	2370	+ cond_resched();
	2371	+ }
187	2372	return 0;
188	2373	}
189	2374
190		-static int ahash_partial_update(struct ahash_request **preq,
191		- struct crypto_ahash tfm, const struct hash_testvec template,
192		- void hash_buff, int k, int temp, struct scatterlist sg,
193		- const char algo, char result, struct crypto_wait *wait)
	2375	+/*
	2376	+ * Test the AEAD algorithm against the corresponding generic implementation, if
	2377	+ * one is available.
	2378	+ */
	2379	+static int test_aead_vs_generic_impl(struct aead_extra_tests_ctx *ctx)
194	2380	{
195		- char *state;
196		- struct ahash_request *req;
197		- int statesize, ret = -EINVAL;
198		- static const unsigned char guard[] = { 0x00, 0xba, 0xad, 0x00 };
199		- int digestsize = crypto_ahash_digestsize(tfm);
	2381	+ struct crypto_aead *tfm = ctx->tfm;
	2382	+ const char *algname = crypto_aead_alg(tfm)->base.cra_name;
	2383	+ const char *driver = ctx->driver;
	2384	+ const char *generic_driver = ctx->test_desc->generic_driver;
	2385	+ char _generic_driver[CRYPTO_MAX_ALG_NAME];
	2386	+ struct crypto_aead *generic_tfm = NULL;
	2387	+ struct aead_request *generic_req = NULL;
	2388	+ unsigned int i;
	2389	+ int err;
200	2390
201		- req = *preq;
202		- statesize = crypto_ahash_statesize(
203		- crypto_ahash_reqtfm(req));
204		- state = kmalloc(statesize + sizeof(guard), GFP_KERNEL);
205		- if (!state) {
206		- pr_err("alg: hash: Failed to alloc state for %s\n", algo);
207		- goto out_nostate;
	2391	+ if (!generic_driver) { /* Use default naming convention? */
	2392	+ err = build_generic_driver_name(algname, _generic_driver);
	2393	+ if (err)
	2394	+ return err;
	2395	+ generic_driver = _generic_driver;
208	2396	}
209		- memcpy(state + statesize, guard, sizeof(guard));
210		- memset(result, 1, digestsize);
211		- ret = crypto_ahash_export(req, state);
212		- WARN_ON(memcmp(state + statesize, guard, sizeof(guard)));
213		- if (ret) {
214		- pr_err("alg: hash: Failed to export() for %s\n", algo);
215		- goto out;
216		- }
217		- ret = ahash_guard_result(result, 1, digestsize);
218		- if (ret) {
219		- pr_err("alg: hash: Failed, export used req->result for %s\n",
220		- algo);
221		- goto out;
222		- }
223		- ahash_request_free(req);
224		- req = ahash_request_alloc(tfm, GFP_KERNEL);
225		- if (!req) {
226		- pr_err("alg: hash: Failed to alloc request for %s\n", algo);
227		- goto out_noreq;
228		- }
229		- ahash_request_set_callback(req,
230		- CRYPTO_TFM_REQ_MAY_BACKLOG,
231		- crypto_req_done, wait);
232	2397
233		- memcpy(hash_buff, template->plaintext + temp,
234		- template->tap[k]);
235		- sg_init_one(&sg[0], hash_buff, template->tap[k]);
236		- ahash_request_set_crypt(req, sg, result, template->tap[k]);
237		- ret = crypto_ahash_import(req, state);
238		- if (ret) {
239		- pr_err("alg: hash: Failed to import() for %s\n", algo);
	2398	+ if (strcmp(generic_driver, driver) == 0) /* Already the generic impl? */
	2399	+ return 0;
	2400	+
	2401	+ generic_tfm = crypto_alloc_aead(generic_driver, 0, 0);
	2402	+ if (IS_ERR(generic_tfm)) {
	2403	+ err = PTR_ERR(generic_tfm);
	2404	+ if (err == -ENOENT) {
	2405	+ pr_warn("alg: aead: skipping comparison tests for %s because %s is unavailable\n",
	2406	+ driver, generic_driver);
	2407	+ return 0;
	2408	+ }
	2409	+ pr_err("alg: aead: error allocating %s (generic impl of %s): %d\n",
	2410	+ generic_driver, algname, err);
	2411	+ return err;
	2412	+ }
	2413	+
	2414	+ generic_req = aead_request_alloc(generic_tfm, GFP_KERNEL);
	2415	+ if (!generic_req) {
	2416	+ err = -ENOMEM;
240	2417	goto out;
241	2418	}
242		- ret = ahash_guard_result(result, 1, digestsize);
243		- if (ret) {
244		- pr_err("alg: hash: Failed, import used req->result for %s\n",
245		- algo);
	2419	+
	2420	+ /* Check the algorithm properties for consistency. */
	2421	+
	2422	+ if (crypto_aead_maxauthsize(tfm) !=
	2423	+ crypto_aead_maxauthsize(generic_tfm)) {
	2424	+ pr_err("alg: aead: maxauthsize for %s (%u) doesn't match generic impl (%u)\n",
	2425	+ driver, crypto_aead_maxauthsize(tfm),
	2426	+ crypto_aead_maxauthsize(generic_tfm));
	2427	+ err = -EINVAL;
246	2428	goto out;
247	2429	}
248		- ret = crypto_wait_req(crypto_ahash_update(req), wait);
249		- if (ret)
	2430	+
	2431	+ if (crypto_aead_ivsize(tfm) != crypto_aead_ivsize(generic_tfm)) {
	2432	+ pr_err("alg: aead: ivsize for %s (%u) doesn't match generic impl (%u)\n",
	2433	+ driver, crypto_aead_ivsize(tfm),
	2434	+ crypto_aead_ivsize(generic_tfm));
	2435	+ err = -EINVAL;
250	2436	goto out;
251		- *preq = req;
252		- ret = 0;
253		- goto out_noreq;
	2437	+ }
	2438	+
	2439	+ if (crypto_aead_blocksize(tfm) != crypto_aead_blocksize(generic_tfm)) {
	2440	+ pr_err("alg: aead: blocksize for %s (%u) doesn't match generic impl (%u)\n",
	2441	+ driver, crypto_aead_blocksize(tfm),
	2442	+ crypto_aead_blocksize(generic_tfm));
	2443	+ err = -EINVAL;
	2444	+ goto out;
	2445	+ }
	2446	+
	2447	+ /*
	2448	+ * Now generate test vectors using the generic implementation, and test
	2449	+ * the other implementation against them.
	2450	+ */
	2451	+ for (i = 0; i < fuzz_iterations * 8; i++) {
	2452	+ generate_random_aead_testvec(generic_req, &ctx->vec,
	2453	+ &ctx->test_desc->suite.aead,
	2454	+ ctx->maxkeysize, ctx->maxdatasize,
	2455	+ ctx->vec_name,
	2456	+ sizeof(ctx->vec_name), false);
	2457	+ generate_random_testvec_config(&ctx->cfg, ctx->cfgname,
	2458	+ sizeof(ctx->cfgname));
	2459	+ if (!ctx->vec.novrfy) {
	2460	+ err = test_aead_vec_cfg(driver, ENCRYPT, &ctx->vec,
	2461	+ ctx->vec_name, &ctx->cfg,
	2462	+ ctx->req, ctx->tsgls);
	2463	+ if (err)
	2464	+ goto out;
	2465	+ }
	2466	+ if (ctx->vec.crypt_error == 0 \|\| ctx->vec.novrfy) {
	2467	+ err = test_aead_vec_cfg(driver, DECRYPT, &ctx->vec,
	2468	+ ctx->vec_name, &ctx->cfg,
	2469	+ ctx->req, ctx->tsgls);
	2470	+ if (err)
	2471	+ goto out;
	2472	+ }
	2473	+ cond_resched();
	2474	+ }
	2475	+ err = 0;
254	2476	out:
255		- ahash_request_free(req);
256		-out_noreq:
257		- kfree(state);
258		-out_nostate:
259		- return ret;
	2477	+ crypto_free_aead(generic_tfm);
	2478	+ aead_request_free(generic_req);
	2479	+ return err;
260	2480	}
261	2481
262		-enum hash_test {
263		- HASH_TEST_DIGEST,
264		- HASH_TEST_FINAL,
265		- HASH_TEST_FINUP
266		-};
267		-
268		-static int __test_hash(struct crypto_ahash *tfm,
269		- const struct hash_testvec *template, unsigned int tcount,
270		- enum hash_test test_type, const int align_offset)
	2482	+static int test_aead_extra(const char *driver,
	2483	+ const struct alg_test_desc *test_desc,
	2484	+ struct aead_request *req,
	2485	+ struct cipher_test_sglists *tsgls)
271	2486	{
272		- const char *algo = crypto_tfm_alg_driver_name(crypto_ahash_tfm(tfm));
273		- size_t digest_size = crypto_ahash_digestsize(tfm);
274		- unsigned int i, j, k, temp;
275		- struct scatterlist sg[8];
276		- char *result;
277		- char *key;
278		- struct ahash_request *req;
279		- struct crypto_wait wait;
280		- void *hash_buff;
281		- char *xbuf[XBUFSIZE];
282		- int ret = -ENOMEM;
	2487	+ struct aead_extra_tests_ctx *ctx;
	2488	+ unsigned int i;
	2489	+ int err;
283	2490
284		- result = kmalloc(digest_size, GFP_KERNEL);
285		- if (!result)
286		- return ret;
287		- key = kmalloc(MAX_KEYLEN, GFP_KERNEL);
288		- if (!key)
289		- goto out_nobuf;
290		- if (testmgr_alloc_buf(xbuf))
291		- goto out_nobuf;
	2491	+ if (noextratests)
	2492	+ return 0;
292	2493
293		- crypto_init_wait(&wait);
	2494	+ ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
	2495	+ if (!ctx)
	2496	+ return -ENOMEM;
	2497	+ ctx->req = req;
	2498	+ ctx->tfm = crypto_aead_reqtfm(req);
	2499	+ ctx->driver = driver;
	2500	+ ctx->test_desc = test_desc;
	2501	+ ctx->tsgls = tsgls;
	2502	+ ctx->maxdatasize = (2 * PAGE_SIZE) - TESTMGR_POISON_LEN;
	2503	+ ctx->maxkeysize = 0;
	2504	+ for (i = 0; i < test_desc->suite.aead.count; i++)
	2505	+ ctx->maxkeysize = max_t(unsigned int, ctx->maxkeysize,
	2506	+ test_desc->suite.aead.vecs[i].klen);
294	2507
295		- req = ahash_request_alloc(tfm, GFP_KERNEL);
296		- if (!req) {
297		- printk(KERN_ERR "alg: hash: Failed to allocate request for "
298		- "%s\n", algo);
299		- goto out_noreq;
300		- }
301		- ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
302		- crypto_req_done, &wait);
303		-
304		- j = 0;
305		- for (i = 0; i < tcount; i++) {
306		- if (template[i].np)
307		- continue;
308		-
309		- ret = -EINVAL;
310		- if (WARN_ON(align_offset + template[i].psize > PAGE_SIZE))
311		- goto out;
312		-
313		- j++;
314		- memset(result, 0, digest_size);
315		-
316		- hash_buff = xbuf[0];
317		- hash_buff += align_offset;
318		-
319		- memcpy(hash_buff, template[i].plaintext, template[i].psize);
320		- sg_init_one(&sg[0], hash_buff, template[i].psize);
321		-
322		- if (template[i].ksize) {
323		- crypto_ahash_clear_flags(tfm, ~0);
324		- if (template[i].ksize > MAX_KEYLEN) {
325		- pr_err("alg: hash: setkey failed on test %d for %s: key size %d > %d\n",
326		- j, algo, template[i].ksize, MAX_KEYLEN);
327		- ret = -EINVAL;
328		- goto out;
329		- }
330		- memcpy(key, template[i].key, template[i].ksize);
331		- ret = crypto_ahash_setkey(tfm, key, template[i].ksize);
332		- if (ret) {
333		- printk(KERN_ERR "alg: hash: setkey failed on "
334		- "test %d for %s: ret=%d\n", j, algo,
335		- -ret);
336		- goto out;
337		- }
338		- }
339		-
340		- ahash_request_set_crypt(req, sg, result, template[i].psize);
341		- switch (test_type) {
342		- case HASH_TEST_DIGEST:
343		- ret = crypto_wait_req(crypto_ahash_digest(req), &wait);
344		- if (ret) {
345		- pr_err("alg: hash: digest failed on test %d "
346		- "for %s: ret=%d\n", j, algo, -ret);
347		- goto out;
348		- }
349		- break;
350		-
351		- case HASH_TEST_FINAL:
352		- memset(result, 1, digest_size);
353		- ret = crypto_wait_req(crypto_ahash_init(req), &wait);
354		- if (ret) {
355		- pr_err("alg: hash: init failed on test %d "
356		- "for %s: ret=%d\n", j, algo, -ret);
357		- goto out;
358		- }
359		- ret = ahash_guard_result(result, 1, digest_size);
360		- if (ret) {
361		- pr_err("alg: hash: init failed on test %d "
362		- "for %s: used req->result\n", j, algo);
363		- goto out;
364		- }
365		- ret = crypto_wait_req(crypto_ahash_update(req), &wait);
366		- if (ret) {
367		- pr_err("alg: hash: update failed on test %d "
368		- "for %s: ret=%d\n", j, algo, -ret);
369		- goto out;
370		- }
371		- ret = ahash_guard_result(result, 1, digest_size);
372		- if (ret) {
373		- pr_err("alg: hash: update failed on test %d "
374		- "for %s: used req->result\n", j, algo);
375		- goto out;
376		- }
377		- ret = crypto_wait_req(crypto_ahash_final(req), &wait);
378		- if (ret) {
379		- pr_err("alg: hash: final failed on test %d "
380		- "for %s: ret=%d\n", j, algo, -ret);
381		- goto out;
382		- }
383		- break;
384		-
385		- case HASH_TEST_FINUP:
386		- memset(result, 1, digest_size);
387		- ret = crypto_wait_req(crypto_ahash_init(req), &wait);
388		- if (ret) {
389		- pr_err("alg: hash: init failed on test %d "
390		- "for %s: ret=%d\n", j, algo, -ret);
391		- goto out;
392		- }
393		- ret = ahash_guard_result(result, 1, digest_size);
394		- if (ret) {
395		- pr_err("alg: hash: init failed on test %d "
396		- "for %s: used req->result\n", j, algo);
397		- goto out;
398		- }
399		- ret = crypto_wait_req(crypto_ahash_finup(req), &wait);
400		- if (ret) {
401		- pr_err("alg: hash: final failed on test %d "
402		- "for %s: ret=%d\n", j, algo, -ret);
403		- goto out;
404		- }
405		- break;
406		- }
407		-
408		- if (memcmp(result, template[i].digest,
409		- crypto_ahash_digestsize(tfm))) {
410		- printk(KERN_ERR "alg: hash: Test %d failed for %s\n",
411		- j, algo);
412		- hexdump(result, crypto_ahash_digestsize(tfm));
413		- ret = -EINVAL;
414		- goto out;
415		- }
	2508	+ ctx->vec.key = kmalloc(ctx->maxkeysize, GFP_KERNEL);
	2509	+ ctx->vec.iv = kmalloc(crypto_aead_ivsize(ctx->tfm), GFP_KERNEL);
	2510	+ ctx->vec.assoc = kmalloc(ctx->maxdatasize, GFP_KERNEL);
	2511	+ ctx->vec.ptext = kmalloc(ctx->maxdatasize, GFP_KERNEL);
	2512	+ ctx->vec.ctext = kmalloc(ctx->maxdatasize, GFP_KERNEL);
	2513	+ if (!ctx->vec.key \|\| !ctx->vec.iv \|\| !ctx->vec.assoc \|\|
	2514	+ !ctx->vec.ptext \|\| !ctx->vec.ctext) {
	2515	+ err = -ENOMEM;
	2516	+ goto out;
416	2517	}
417	2518
418		- if (test_type)
	2519	+ err = test_aead_vs_generic_impl(ctx);
	2520	+ if (err)
419	2521	goto out;
420	2522
421		- j = 0;
422		- for (i = 0; i < tcount; i++) {
423		- /* alignment tests are only done with continuous buffers */
424		- if (align_offset != 0)
425		- break;
426		-
427		- if (!template[i].np)
428		- continue;
429		-
430		- j++;
431		- memset(result, 0, digest_size);
432		-
433		- temp = 0;
434		- sg_init_table(sg, template[i].np);
435		- ret = -EINVAL;
436		- for (k = 0; k < template[i].np; k++) {
437		- if (WARN_ON(offset_in_page(IDX[k]) +
438		- template[i].tap[k] > PAGE_SIZE))
439		- goto out;
440		- sg_set_buf(&sg[k],
441		- memcpy(xbuf[IDX[k] >> PAGE_SHIFT] +
442		- offset_in_page(IDX[k]),
443		- template[i].plaintext + temp,
444		- template[i].tap[k]),
445		- template[i].tap[k]);
446		- temp += template[i].tap[k];
447		- }
448		-
449		- if (template[i].ksize) {
450		- if (template[i].ksize > MAX_KEYLEN) {
451		- pr_err("alg: hash: setkey failed on test %d for %s: key size %d > %d\n",
452		- j, algo, template[i].ksize, MAX_KEYLEN);
453		- ret = -EINVAL;
454		- goto out;
455		- }
456		- crypto_ahash_clear_flags(tfm, ~0);
457		- memcpy(key, template[i].key, template[i].ksize);
458		- ret = crypto_ahash_setkey(tfm, key, template[i].ksize);
459		-
460		- if (ret) {
461		- printk(KERN_ERR "alg: hash: setkey "
462		- "failed on chunking test %d "
463		- "for %s: ret=%d\n", j, algo, -ret);
464		- goto out;
465		- }
466		- }
467		-
468		- ahash_request_set_crypt(req, sg, result, template[i].psize);
469		- ret = crypto_wait_req(crypto_ahash_digest(req), &wait);
470		- if (ret) {
471		- pr_err("alg: hash: digest failed on chunking test %d for %s: ret=%d\n",
472		- j, algo, -ret);
473		- goto out;
474		- }
475		-
476		- if (memcmp(result, template[i].digest,
477		- crypto_ahash_digestsize(tfm))) {
478		- printk(KERN_ERR "alg: hash: Chunking test %d "
479		- "failed for %s\n", j, algo);
480		- hexdump(result, crypto_ahash_digestsize(tfm));
481		- ret = -EINVAL;
482		- goto out;
483		- }
484		- }
485		-
486		- /* partial update exercise */
487		- j = 0;
488		- for (i = 0; i < tcount; i++) {
489		- /* alignment tests are only done with continuous buffers */
490		- if (align_offset != 0)
491		- break;
492		-
493		- if (template[i].np < 2)
494		- continue;
495		-
496		- j++;
497		- memset(result, 0, digest_size);
498		-
499		- ret = -EINVAL;
500		- hash_buff = xbuf[0];
501		- memcpy(hash_buff, template[i].plaintext,
502		- template[i].tap[0]);
503		- sg_init_one(&sg[0], hash_buff, template[i].tap[0]);
504		-
505		- if (template[i].ksize) {
506		- crypto_ahash_clear_flags(tfm, ~0);
507		- if (template[i].ksize > MAX_KEYLEN) {
508		- pr_err("alg: hash: setkey failed on test %d for %s: key size %d > %d\n",
509		- j, algo, template[i].ksize, MAX_KEYLEN);
510		- ret = -EINVAL;
511		- goto out;
512		- }
513		- memcpy(key, template[i].key, template[i].ksize);
514		- ret = crypto_ahash_setkey(tfm, key, template[i].ksize);
515		- if (ret) {
516		- pr_err("alg: hash: setkey failed on test %d for %s: ret=%d\n",
517		- j, algo, -ret);
518		- goto out;
519		- }
520		- }
521		-
522		- ahash_request_set_crypt(req, sg, result, template[i].tap[0]);
523		- ret = crypto_wait_req(crypto_ahash_init(req), &wait);
524		- if (ret) {
525		- pr_err("alg: hash: init failed on test %d for %s: ret=%d\n",
526		- j, algo, -ret);
527		- goto out;
528		- }
529		- ret = crypto_wait_req(crypto_ahash_update(req), &wait);
530		- if (ret) {
531		- pr_err("alg: hash: update failed on test %d for %s: ret=%d\n",
532		- j, algo, -ret);
533		- goto out;
534		- }
535		-
536		- temp = template[i].tap[0];
537		- for (k = 1; k < template[i].np; k++) {
538		- ret = ahash_partial_update(&req, tfm, &template[i],
539		- hash_buff, k, temp, &sg[0], algo, result,
540		- &wait);
541		- if (ret) {
542		- pr_err("alg: hash: partial update failed on test %d for %s: ret=%d\n",
543		- j, algo, -ret);
544		- goto out_noreq;
545		- }
546		- temp += template[i].tap[k];
547		- }
548		- ret = crypto_wait_req(crypto_ahash_final(req), &wait);
549		- if (ret) {
550		- pr_err("alg: hash: final failed on test %d for %s: ret=%d\n",
551		- j, algo, -ret);
552		- goto out;
553		- }
554		- if (memcmp(result, template[i].digest,
555		- crypto_ahash_digestsize(tfm))) {
556		- pr_err("alg: hash: Partial Test %d failed for %s\n",
557		- j, algo);
558		- hexdump(result, crypto_ahash_digestsize(tfm));
559		- ret = -EINVAL;
560		- goto out;
561		- }
562		- }
563		-
564		- ret = 0;
565		-
	2523	+ err = test_aead_inauthentic_inputs(ctx);
566	2524	out:
567		- ahash_request_free(req);
568		-out_noreq:
569		- testmgr_free_buf(xbuf);
570		-out_nobuf:
571		- kfree(key);
572		- kfree(result);
573		- return ret;
	2525	+ kfree(ctx->vec.key);
	2526	+ kfree(ctx->vec.iv);
	2527	+ kfree(ctx->vec.assoc);
	2528	+ kfree(ctx->vec.ptext);
	2529	+ kfree(ctx->vec.ctext);
	2530	+ kfree(ctx);
	2531	+ return err;
574	2532	}
575		-
576		-static int test_hash(struct crypto_ahash *tfm,
577		- const struct hash_testvec *template,
578		- unsigned int tcount, enum hash_test test_type)
	2533	+#else /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */
	2534	+static int test_aead_extra(const char *driver,
	2535	+ const struct alg_test_desc *test_desc,
	2536	+ struct aead_request *req,
	2537	+ struct cipher_test_sglists *tsgls)
579	2538	{
580		- unsigned int alignmask;
581		- int ret;
	2539	+ return 0;
	2540	+}
	2541	+#endif /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */
582	2542
583		- ret = __test_hash(tfm, template, tcount, test_type, 0);
584		- if (ret)
585		- return ret;
	2543	+static int test_aead(const char *driver, int enc,
	2544	+ const struct aead_test_suite *suite,
	2545	+ struct aead_request *req,
	2546	+ struct cipher_test_sglists *tsgls)
	2547	+{
	2548	+ unsigned int i;
	2549	+ int err;
586	2550
587		- /* test unaligned buffers, check with one byte offset */
588		- ret = __test_hash(tfm, template, tcount, test_type, 1);
589		- if (ret)
590		- return ret;
591		-
592		- alignmask = crypto_tfm_alg_alignmask(&tfm->base);
593		- if (alignmask) {
594		- /* Check if alignment mask for tfm is correctly set. */
595		- ret = __test_hash(tfm, template, tcount, test_type,
596		- alignmask + 1);
597		- if (ret)
598		- return ret;
	2551	+ for (i = 0; i < suite->count; i++) {
	2552	+ err = test_aead_vec(driver, enc, &suite->vecs[i], i, req,
	2553	+ tsgls);
	2554	+ if (err)
	2555	+ return err;
	2556	+ cond_resched();
599	2557	}
600		-
601	2558	return 0;
602	2559	}
603	2560
604		-static int __test_aead(struct crypto_aead *tfm, int enc,
605		- const struct aead_testvec *template, unsigned int tcount,
606		- const bool diff_dst, const int align_offset)
	2561	+static int alg_test_aead(const struct alg_test_desc desc, const char driver,
	2562	+ u32 type, u32 mask)
607	2563	{
608		- const char *algo = crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm));
609		- unsigned int i, j, k, n, temp;
610		- int ret = -ENOMEM;
611		- char *q;
612		- char *key;
613		- struct aead_request *req;
614		- struct scatterlist *sg;
615		- struct scatterlist *sgout;
616		- const char e, d;
617		- struct crypto_wait wait;
618		- unsigned int authsize, iv_len;
619		- void *input;
620		- void *output;
621		- void *assoc;
622		- char *iv;
623		- char *xbuf[XBUFSIZE];
624		- char *xoutbuf[XBUFSIZE];
625		- char *axbuf[XBUFSIZE];
	2564	+ const struct aead_test_suite *suite = &desc->suite.aead;
	2565	+ struct crypto_aead *tfm;
	2566	+ struct aead_request *req = NULL;
	2567	+ struct cipher_test_sglists *tsgls = NULL;
	2568	+ int err;
626	2569
627		- iv = kzalloc(MAX_IVLEN, GFP_KERNEL);
628		- if (!iv)
629		- return ret;
630		- key = kmalloc(MAX_KEYLEN, GFP_KERNEL);
631		- if (!key)
632		- goto out_noxbuf;
633		- if (testmgr_alloc_buf(xbuf))
634		- goto out_noxbuf;
635		- if (testmgr_alloc_buf(axbuf))
636		- goto out_noaxbuf;
637		- if (diff_dst && testmgr_alloc_buf(xoutbuf))
638		- goto out_nooutbuf;
	2570	+ if (suite->count <= 0) {
	2571	+ pr_err("alg: aead: empty test suite for %s\n", driver);
	2572	+ return -EINVAL;
	2573	+ }
639	2574
640		- /* avoid "the frame size is larger than 1024 bytes" compiler warning */
641		- sg = kmalloc(array3_size(sizeof(*sg), 8, (diff_dst ? 4 : 2)),
642		- GFP_KERNEL);
643		- if (!sg)
644		- goto out_nosg;
645		- sgout = &sg[16];
646		-
647		- if (diff_dst)
648		- d = "-ddst";
649		- else
650		- d = "";
651		-
652		- if (enc == ENCRYPT)
653		- e = "encryption";
654		- else
655		- e = "decryption";
656		-
657		- crypto_init_wait(&wait);
	2575	+ tfm = crypto_alloc_aead(driver, type, mask);
	2576	+ if (IS_ERR(tfm)) {
	2577	+ pr_err("alg: aead: failed to allocate transform for %s: %ld\n",
	2578	+ driver, PTR_ERR(tfm));
	2579	+ return PTR_ERR(tfm);
	2580	+ }
658	2581
659	2582	req = aead_request_alloc(tfm, GFP_KERNEL);
660	2583	if (!req) {
661		- pr_err("alg: aead%s: Failed to allocate request for %s\n",
662		- d, algo);
	2584	+ pr_err("alg: aead: failed to allocate request for %s\n",
	2585	+ driver);
	2586	+ err = -ENOMEM;
663	2587	goto out;
664	2588	}
665	2589
666		- aead_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
667		- crypto_req_done, &wait);
668		-
669		- iv_len = crypto_aead_ivsize(tfm);
670		-
671		- for (i = 0, j = 0; i < tcount; i++) {
672		- if (template[i].np)
673		- continue;
674		-
675		- j++;
676		-
677		- /* some templates have no input data but they will
678		- * touch input
679		- */
680		- input = xbuf[0];
681		- input += align_offset;
682		- assoc = axbuf[0];
683		-
684		- ret = -EINVAL;
685		- if (WARN_ON(align_offset + template[i].ilen >
686		- PAGE_SIZE \|\| template[i].alen > PAGE_SIZE))
687		- goto out;
688		-
689		- memcpy(input, template[i].input, template[i].ilen);
690		- memcpy(assoc, template[i].assoc, template[i].alen);
691		- if (template[i].iv)
692		- memcpy(iv, template[i].iv, iv_len);
693		- else
694		- memset(iv, 0, iv_len);
695		-
696		- crypto_aead_clear_flags(tfm, ~0);
697		- if (template[i].wk)
698		- crypto_aead_set_flags(tfm, CRYPTO_TFM_REQ_WEAK_KEY);
699		-
700		- if (template[i].klen > MAX_KEYLEN) {
701		- pr_err("alg: aead%s: setkey failed on test %d for %s: key size %d > %d\n",
702		- d, j, algo, template[i].klen,
703		- MAX_KEYLEN);
704		- ret = -EINVAL;
705		- goto out;
706		- }
707		- memcpy(key, template[i].key, template[i].klen);
708		-
709		- ret = crypto_aead_setkey(tfm, key, template[i].klen);
710		- if (template[i].fail == !ret) {
711		- pr_err("alg: aead%s: setkey failed on test %d for %s: flags=%x\n",
712		- d, j, algo, crypto_aead_get_flags(tfm));
713		- goto out;
714		- } else if (ret)
715		- continue;
716		-
717		- authsize = abs(template[i].rlen - template[i].ilen);
718		- ret = crypto_aead_setauthsize(tfm, authsize);
719		- if (ret) {
720		- pr_err("alg: aead%s: Failed to set authsize to %u on test %d for %s\n",
721		- d, authsize, j, algo);
722		- goto out;
723		- }
724		-
725		- k = !!template[i].alen;
726		- sg_init_table(sg, k + 1);
727		- sg_set_buf(&sg[0], assoc, template[i].alen);
728		- sg_set_buf(&sg[k], input,
729		- template[i].ilen + (enc ? authsize : 0));
730		- output = input;
731		-
732		- if (diff_dst) {
733		- sg_init_table(sgout, k + 1);
734		- sg_set_buf(&sgout[0], assoc, template[i].alen);
735		-
736		- output = xoutbuf[0];
737		- output += align_offset;
738		- sg_set_buf(&sgout[k], output,
739		- template[i].rlen + (enc ? 0 : authsize));
740		- }
741		-
742		- aead_request_set_crypt(req, sg, (diff_dst) ? sgout : sg,
743		- template[i].ilen, iv);
744		-
745		- aead_request_set_ad(req, template[i].alen);
746		-
747		- ret = crypto_wait_req(enc ? crypto_aead_encrypt(req)
748		- : crypto_aead_decrypt(req), &wait);
749		-
750		- switch (ret) {
751		- case 0:
752		- if (template[i].novrfy) {
753		- /* verification was supposed to fail */
754		- pr_err("alg: aead%s: %s failed on test %d for %s: ret was 0, expected -EBADMSG\n",
755		- d, e, j, algo);
756		- /* so really, we got a bad message */
757		- ret = -EBADMSG;
758		- goto out;
759		- }
760		- break;
761		- case -EBADMSG:
762		- if (template[i].novrfy)
763		- /* verification failure was expected */
764		- continue;
765		- /* fall through */
766		- default:
767		- pr_err("alg: aead%s: %s failed on test %d for %s: ret=%d\n",
768		- d, e, j, algo, -ret);
769		- goto out;
770		- }
771		-
772		- q = output;
773		- if (memcmp(q, template[i].result, template[i].rlen)) {
774		- pr_err("alg: aead%s: Test %d failed on %s for %s\n",
775		- d, j, e, algo);
776		- hexdump(q, template[i].rlen);
777		- ret = -EINVAL;
778		- goto out;
779		- }
	2590	+ tsgls = alloc_cipher_test_sglists();
	2591	+ if (!tsgls) {
	2592	+ pr_err("alg: aead: failed to allocate test buffers for %s\n",
	2593	+ driver);
	2594	+ err = -ENOMEM;
	2595	+ goto out;
780	2596	}
781	2597
782		- for (i = 0, j = 0; i < tcount; i++) {
783		- /* alignment tests are only done with continuous buffers */
784		- if (align_offset != 0)
785		- break;
	2598	+ err = test_aead(driver, ENCRYPT, suite, req, tsgls);
	2599	+ if (err)
	2600	+ goto out;
786	2601
787		- if (!template[i].np)
788		- continue;
	2602	+ err = test_aead(driver, DECRYPT, suite, req, tsgls);
	2603	+ if (err)
	2604	+ goto out;
789	2605
790		- j++;
791		-
792		- if (template[i].iv)
793		- memcpy(iv, template[i].iv, iv_len);
794		- else
795		- memset(iv, 0, MAX_IVLEN);
796		-
797		- crypto_aead_clear_flags(tfm, ~0);
798		- if (template[i].wk)
799		- crypto_aead_set_flags(tfm, CRYPTO_TFM_REQ_WEAK_KEY);
800		- if (template[i].klen > MAX_KEYLEN) {
801		- pr_err("alg: aead%s: setkey failed on test %d for %s: key size %d > %d\n",
802		- d, j, algo, template[i].klen, MAX_KEYLEN);
803		- ret = -EINVAL;
804		- goto out;
805		- }
806		- memcpy(key, template[i].key, template[i].klen);
807		-
808		- ret = crypto_aead_setkey(tfm, key, template[i].klen);
809		- if (template[i].fail == !ret) {
810		- pr_err("alg: aead%s: setkey failed on chunk test %d for %s: flags=%x\n",
811		- d, j, algo, crypto_aead_get_flags(tfm));
812		- goto out;
813		- } else if (ret)
814		- continue;
815		-
816		- authsize = abs(template[i].rlen - template[i].ilen);
817		-
818		- ret = -EINVAL;
819		- sg_init_table(sg, template[i].anp + template[i].np);
820		- if (diff_dst)
821		- sg_init_table(sgout, template[i].anp + template[i].np);
822		-
823		- ret = -EINVAL;
824		- for (k = 0, temp = 0; k < template[i].anp; k++) {
825		- if (WARN_ON(offset_in_page(IDX[k]) +
826		- template[i].atap[k] > PAGE_SIZE))
827		- goto out;
828		- sg_set_buf(&sg[k],
829		- memcpy(axbuf[IDX[k] >> PAGE_SHIFT] +
830		- offset_in_page(IDX[k]),
831		- template[i].assoc + temp,
832		- template[i].atap[k]),
833		- template[i].atap[k]);
834		- if (diff_dst)
835		- sg_set_buf(&sgout[k],
836		- axbuf[IDX[k] >> PAGE_SHIFT] +
837		- offset_in_page(IDX[k]),
838		- template[i].atap[k]);
839		- temp += template[i].atap[k];
840		- }
841		-
842		- for (k = 0, temp = 0; k < template[i].np; k++) {
843		- if (WARN_ON(offset_in_page(IDX[k]) +
844		- template[i].tap[k] > PAGE_SIZE))
845		- goto out;
846		-
847		- q = xbuf[IDX[k] >> PAGE_SHIFT] + offset_in_page(IDX[k]);
848		- memcpy(q, template[i].input + temp, template[i].tap[k]);
849		- sg_set_buf(&sg[template[i].anp + k],
850		- q, template[i].tap[k]);
851		-
852		- if (diff_dst) {
853		- q = xoutbuf[IDX[k] >> PAGE_SHIFT] +
854		- offset_in_page(IDX[k]);
855		-
856		- memset(q, 0, template[i].tap[k]);
857		-
858		- sg_set_buf(&sgout[template[i].anp + k],
859		- q, template[i].tap[k]);
860		- }
861		-
862		- n = template[i].tap[k];
863		- if (k == template[i].np - 1 && enc)
864		- n += authsize;
865		- if (offset_in_page(q) + n < PAGE_SIZE)
866		- q[n] = 0;
867		-
868		- temp += template[i].tap[k];
869		- }
870		-
871		- ret = crypto_aead_setauthsize(tfm, authsize);
872		- if (ret) {
873		- pr_err("alg: aead%s: Failed to set authsize to %u on chunk test %d for %s\n",
874		- d, authsize, j, algo);
875		- goto out;
876		- }
877		-
878		- if (enc) {
879		- if (WARN_ON(sg[template[i].anp + k - 1].offset +
880		- sg[template[i].anp + k - 1].length +
881		- authsize > PAGE_SIZE)) {
882		- ret = -EINVAL;
883		- goto out;
884		- }
885		-
886		- if (diff_dst)
887		- sgout[template[i].anp + k - 1].length +=
888		- authsize;
889		- sg[template[i].anp + k - 1].length += authsize;
890		- }
891		-
892		- aead_request_set_crypt(req, sg, (diff_dst) ? sgout : sg,
893		- template[i].ilen,
894		- iv);
895		-
896		- aead_request_set_ad(req, template[i].alen);
897		-
898		- ret = crypto_wait_req(enc ? crypto_aead_encrypt(req)
899		- : crypto_aead_decrypt(req), &wait);
900		-
901		- switch (ret) {
902		- case 0:
903		- if (template[i].novrfy) {
904		- /* verification was supposed to fail */
905		- pr_err("alg: aead%s: %s failed on chunk test %d for %s: ret was 0, expected -EBADMSG\n",
906		- d, e, j, algo);
907		- /* so really, we got a bad message */
908		- ret = -EBADMSG;
909		- goto out;
910		- }
911		- break;
912		- case -EBADMSG:
913		- if (template[i].novrfy)
914		- /* verification failure was expected */
915		- continue;
916		- /* fall through */
917		- default:
918		- pr_err("alg: aead%s: %s failed on chunk test %d for %s: ret=%d\n",
919		- d, e, j, algo, -ret);
920		- goto out;
921		- }
922		-
923		- ret = -EINVAL;
924		- for (k = 0, temp = 0; k < template[i].np; k++) {
925		- if (diff_dst)
926		- q = xoutbuf[IDX[k] >> PAGE_SHIFT] +
927		- offset_in_page(IDX[k]);
928		- else
929		- q = xbuf[IDX[k] >> PAGE_SHIFT] +
930		- offset_in_page(IDX[k]);
931		-
932		- n = template[i].tap[k];
933		- if (k == template[i].np - 1)
934		- n += enc ? authsize : -authsize;
935		-
936		- if (memcmp(q, template[i].result + temp, n)) {
937		- pr_err("alg: aead%s: Chunk test %d failed on %s at page %u for %s\n",
938		- d, j, e, k, algo);
939		- hexdump(q, n);
940		- goto out;
941		- }
942		-
943		- q += n;
944		- if (k == template[i].np - 1 && !enc) {
945		- if (!diff_dst &&
946		- memcmp(q, template[i].input +
947		- temp + n, authsize))
948		- n = authsize;
949		- else
950		- n = 0;
951		- } else {
952		- for (n = 0; offset_in_page(q + n) && q[n]; n++)
953		- ;
954		- }
955		- if (n) {
956		- pr_err("alg: aead%s: Result buffer corruption in chunk test %d on %s at page %u for %s: %u bytes:\n",
957		- d, j, e, k, algo, n);
958		- hexdump(q, n);
959		- goto out;
960		- }
961		-
962		- temp += template[i].tap[k];
963		- }
964		- }
965		-
966		- ret = 0;
967		-
	2606	+ err = test_aead_extra(driver, desc, req, tsgls);
968	2607	out:
	2608	+ free_cipher_test_sglists(tsgls);
969	2609	aead_request_free(req);
970		- kfree(sg);
971		-out_nosg:
972		- if (diff_dst)
973		- testmgr_free_buf(xoutbuf);
974		-out_nooutbuf:
975		- testmgr_free_buf(axbuf);
976		-out_noaxbuf:
977		- testmgr_free_buf(xbuf);
978		-out_noxbuf:
979		- kfree(key);
980		- kfree(iv);
981		- return ret;
982		-}
983		-
984		-static int test_aead(struct crypto_aead *tfm, int enc,
985		- const struct aead_testvec *template, unsigned int tcount)
986		-{
987		- unsigned int alignmask;
988		- int ret;
989		-
990		- /* test 'dst == src' case */
991		- ret = __test_aead(tfm, enc, template, tcount, false, 0);
992		- if (ret)
993		- return ret;
994		-
995		- /* test 'dst != src' case */
996		- ret = __test_aead(tfm, enc, template, tcount, true, 0);
997		- if (ret)
998		- return ret;
999		-
1000		- /* test unaligned buffers, check with one byte offset */
1001		- ret = __test_aead(tfm, enc, template, tcount, true, 1);
1002		- if (ret)
1003		- return ret;
1004		-
1005		- alignmask = crypto_tfm_alg_alignmask(&tfm->base);
1006		- if (alignmask) {
1007		- /* Check if alignment mask for tfm is correctly set. */
1008		- ret = __test_aead(tfm, enc, template, tcount, true,
1009		- alignmask + 1);
1010		- if (ret)
1011		- return ret;
1012		- }
1013		-
1014		- return 0;
	2610	+ crypto_free_aead(tfm);
	2611	+ return err;
1015	2612	}
1016	2613
1017	2614	static int test_cipher(struct crypto_cipher *tfm, int enc,
..	..	@@ -1037,8 +2634,6 @@
1037	2634
1038	2635	j = 0;
1039	2636	for (i = 0; i < tcount; i++) {
1040		- if (template[i].np)
1041		- continue;
1042	2637
1043	2638	if (fips_enabled && template[i].fips_skip)
1044	2639	continue;
..	..	@@ -1056,17 +2651,24 @@
1056	2651
1057	2652	crypto_cipher_clear_flags(tfm, ~0);
1058	2653	if (template[i].wk)
1059		- crypto_cipher_set_flags(tfm, CRYPTO_TFM_REQ_WEAK_KEY);
	2654	+ crypto_cipher_set_flags(tfm, CRYPTO_TFM_REQ_FORBID_WEAK_KEYS);
1060	2655
1061	2656	ret = crypto_cipher_setkey(tfm, template[i].key,
1062	2657	template[i].klen);
1063		- if (template[i].fail == !ret) {
1064		- printk(KERN_ERR "alg: cipher: setkey failed "
1065		- "on test %d for %s: flags=%x\n", j,
1066		- algo, crypto_cipher_get_flags(tfm));
	2658	+ if (ret) {
	2659	+ if (ret == template[i].setkey_error)
	2660	+ continue;
	2661	+ pr_err("alg: cipher: %s setkey failed on test vector %u; expected_error=%d, actual_error=%d, flags=%#x\n",
	2662	+ algo, j, template[i].setkey_error, ret,
	2663	+ crypto_cipher_get_flags(tfm));
1067	2664	goto out;
1068		- } else if (ret)
1069		- continue;
	2665	+ }
	2666	+ if (template[i].setkey_error) {
	2667	+ pr_err("alg: cipher: %s setkey unexpectedly succeeded on test vector %u; expected_error=%d\n",
	2668	+ algo, j, template[i].setkey_error);
	2669	+ ret = -EINVAL;
	2670	+ goto out;
	2671	+ }
1070	2672
1071	2673	for (k = 0; k < template[i].len;
1072	2674	k += crypto_cipher_blocksize(tfm)) {
..	..	@@ -1096,284 +2698,489 @@
1096	2698	return ret;
1097	2699	}
1098	2700
1099		-static int __test_skcipher(struct crypto_skcipher *tfm, int enc,
1100		- const struct cipher_testvec *template,
1101		- unsigned int tcount,
1102		- const bool diff_dst, const int align_offset)
	2701	+static int test_skcipher_vec_cfg(const char *driver, int enc,
	2702	+ const struct cipher_testvec *vec,
	2703	+ const char *vec_name,
	2704	+ const struct testvec_config *cfg,
	2705	+ struct skcipher_request *req,
	2706	+ struct cipher_test_sglists *tsgls)
1103	2707	{
1104		- const char *algo =
1105		- crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm));
1106		- unsigned int i, j, k, n, temp;
1107		- char *q;
1108		- struct skcipher_request *req;
1109		- struct scatterlist sg[8];
1110		- struct scatterlist sgout[8];
1111		- const char e, d;
1112		- struct crypto_wait wait;
1113		- const char input, result;
1114		- void *data;
1115		- char iv[MAX_IVLEN];
1116		- char *xbuf[XBUFSIZE];
1117		- char *xoutbuf[XBUFSIZE];
1118		- int ret = -ENOMEM;
1119		- unsigned int ivsize = crypto_skcipher_ivsize(tfm);
	2708	+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
	2709	+ const unsigned int alignmask = crypto_skcipher_alignmask(tfm);
	2710	+ const unsigned int ivsize = crypto_skcipher_ivsize(tfm);
	2711	+ const u32 req_flags = CRYPTO_TFM_REQ_MAY_BACKLOG \| cfg->req_flags;
	2712	+ const char *op = enc ? "encryption" : "decryption";
	2713	+ DECLARE_CRYPTO_WAIT(wait);
	2714	+ u8 _iv[3 * (MAX_ALGAPI_ALIGNMASK + 1) + MAX_IVLEN];
	2715	+ u8 iv = PTR_ALIGN(&_iv[0], 2 (MAX_ALGAPI_ALIGNMASK + 1)) +
	2716	+ cfg->iv_offset +
	2717	+ (cfg->iv_offset_relative_to_alignmask ? alignmask : 0);
	2718	+ struct kvec input;
	2719	+ int err;
1120	2720
1121		- if (testmgr_alloc_buf(xbuf))
1122		- goto out_nobuf;
1123		-
1124		- if (diff_dst && testmgr_alloc_buf(xoutbuf))
1125		- goto out_nooutbuf;
1126		-
1127		- if (diff_dst)
1128		- d = "-ddst";
	2721	+ /* Set the key */
	2722	+ if (vec->wk)
	2723	+ crypto_skcipher_set_flags(tfm, CRYPTO_TFM_REQ_FORBID_WEAK_KEYS);
1129	2724	else
1130		- d = "";
1131		-
1132		- if (enc == ENCRYPT)
1133		- e = "encryption";
1134		- else
1135		- e = "decryption";
1136		-
1137		- crypto_init_wait(&wait);
1138		-
1139		- req = skcipher_request_alloc(tfm, GFP_KERNEL);
1140		- if (!req) {
1141		- pr_err("alg: skcipher%s: Failed to allocate request for %s\n",
1142		- d, algo);
1143		- goto out;
	2725	+ crypto_skcipher_clear_flags(tfm,
	2726	+ CRYPTO_TFM_REQ_FORBID_WEAK_KEYS);
	2727	+ err = do_setkey(crypto_skcipher_setkey, tfm, vec->key, vec->klen,
	2728	+ cfg, alignmask);
	2729	+ if (err) {
	2730	+ if (err == vec->setkey_error)
	2731	+ return 0;
	2732	+ pr_err("alg: skcipher: %s setkey failed on test vector %s; expected_error=%d, actual_error=%d, flags=%#x\n",
	2733	+ driver, vec_name, vec->setkey_error, err,
	2734	+ crypto_skcipher_get_flags(tfm));
	2735	+ return err;
	2736	+ }
	2737	+ if (vec->setkey_error) {
	2738	+ pr_err("alg: skcipher: %s setkey unexpectedly succeeded on test vector %s; expected_error=%d\n",
	2739	+ driver, vec_name, vec->setkey_error);
	2740	+ return -EINVAL;
1144	2741	}
1145	2742
1146		- skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
1147		- crypto_req_done, &wait);
1148		-
1149		- j = 0;
1150		- for (i = 0; i < tcount; i++) {
1151		- if (template[i].np && !template[i].also_non_np)
1152		- continue;
1153		-
1154		- if (fips_enabled && template[i].fips_skip)
1155		- continue;
1156		-
1157		- if (template[i].iv && !(template[i].generates_iv && enc))
1158		- memcpy(iv, template[i].iv, ivsize);
	2743	+ /* The IV must be copied to a buffer, as the algorithm may modify it */
	2744	+ if (ivsize) {
	2745	+ if (WARN_ON(ivsize > MAX_IVLEN))
	2746	+ return -EINVAL;
	2747	+ if (vec->generates_iv && !enc)
	2748	+ memcpy(iv, vec->iv_out, ivsize);
	2749	+ else if (vec->iv)
	2750	+ memcpy(iv, vec->iv, ivsize);
1159	2751	else
1160		- memset(iv, 0, MAX_IVLEN);
1161		-
1162		- input = enc ? template[i].ptext : template[i].ctext;
1163		- result = enc ? template[i].ctext : template[i].ptext;
1164		- j++;
1165		- ret = -EINVAL;
1166		- if (WARN_ON(align_offset + template[i].len > PAGE_SIZE))
1167		- goto out;
1168		-
1169		- data = xbuf[0];
1170		- data += align_offset;
1171		- memcpy(data, input, template[i].len);
1172		-
1173		- crypto_skcipher_clear_flags(tfm, ~0);
1174		- if (template[i].wk)
1175		- crypto_skcipher_set_flags(tfm,
1176		- CRYPTO_TFM_REQ_WEAK_KEY);
1177		-
1178		- ret = crypto_skcipher_setkey(tfm, template[i].key,
1179		- template[i].klen);
1180		- if (template[i].fail == !ret) {
1181		- pr_err("alg: skcipher%s: setkey failed on test %d for %s: flags=%x\n",
1182		- d, j, algo, crypto_skcipher_get_flags(tfm));
1183		- goto out;
1184		- } else if (ret)
1185		- continue;
1186		-
1187		- sg_init_one(&sg[0], data, template[i].len);
1188		- if (diff_dst) {
1189		- data = xoutbuf[0];
1190		- data += align_offset;
1191		- sg_init_one(&sgout[0], data, template[i].len);
	2752	+ memset(iv, 0, ivsize);
	2753	+ } else {
	2754	+ if (vec->generates_iv) {
	2755	+ pr_err("alg: skcipher: %s has ivsize=0 but test vector %s generates IV!\n",
	2756	+ driver, vec_name);
	2757	+ return -EINVAL;
1192	2758	}
1193		-
1194		- skcipher_request_set_crypt(req, sg, (diff_dst) ? sgout : sg,
1195		- template[i].len, iv);
1196		- ret = crypto_wait_req(enc ? crypto_skcipher_encrypt(req) :
1197		- crypto_skcipher_decrypt(req), &wait);
1198		-
1199		- if (ret) {
1200		- pr_err("alg: skcipher%s: %s failed on test %d for %s: ret=%d\n",
1201		- d, e, j, algo, -ret);
1202		- goto out;
1203		- }
1204		-
1205		- q = data;
1206		- if (memcmp(q, result, template[i].len)) {
1207		- pr_err("alg: skcipher%s: Test %d failed (invalid result) on %s for %s\n",
1208		- d, j, e, algo);
1209		- hexdump(q, template[i].len);
1210		- ret = -EINVAL;
1211		- goto out;
1212		- }
1213		-
1214		- if (template[i].generates_iv && enc &&
1215		- memcmp(iv, template[i].iv, crypto_skcipher_ivsize(tfm))) {
1216		- pr_err("alg: skcipher%s: Test %d failed (invalid output IV) on %s for %s\n",
1217		- d, j, e, algo);
1218		- hexdump(iv, crypto_skcipher_ivsize(tfm));
1219		- ret = -EINVAL;
1220		- goto out;
1221		- }
	2759	+ iv = NULL;
1222	2760	}
1223	2761
1224		- j = 0;
1225		- for (i = 0; i < tcount; i++) {
1226		- /* alignment tests are only done with continuous buffers */
1227		- if (align_offset != 0)
1228		- break;
1229		-
1230		- if (!template[i].np)
1231		- continue;
1232		-
1233		- if (fips_enabled && template[i].fips_skip)
1234		- continue;
1235		-
1236		- if (template[i].iv && !(template[i].generates_iv && enc))
1237		- memcpy(iv, template[i].iv, ivsize);
1238		- else
1239		- memset(iv, 0, MAX_IVLEN);
1240		-
1241		- input = enc ? template[i].ptext : template[i].ctext;
1242		- result = enc ? template[i].ctext : template[i].ptext;
1243		- j++;
1244		- crypto_skcipher_clear_flags(tfm, ~0);
1245		- if (template[i].wk)
1246		- crypto_skcipher_set_flags(tfm,
1247		- CRYPTO_TFM_REQ_WEAK_KEY);
1248		-
1249		- ret = crypto_skcipher_setkey(tfm, template[i].key,
1250		- template[i].klen);
1251		- if (template[i].fail == !ret) {
1252		- pr_err("alg: skcipher%s: setkey failed on chunk test %d for %s: flags=%x\n",
1253		- d, j, algo, crypto_skcipher_get_flags(tfm));
1254		- goto out;
1255		- } else if (ret)
1256		- continue;
1257		-
1258		- temp = 0;
1259		- ret = -EINVAL;
1260		- sg_init_table(sg, template[i].np);
1261		- if (diff_dst)
1262		- sg_init_table(sgout, template[i].np);
1263		- for (k = 0; k < template[i].np; k++) {
1264		- if (WARN_ON(offset_in_page(IDX[k]) +
1265		- template[i].tap[k] > PAGE_SIZE))
1266		- goto out;
1267		-
1268		- q = xbuf[IDX[k] >> PAGE_SHIFT] + offset_in_page(IDX[k]);
1269		-
1270		- memcpy(q, input + temp, template[i].tap[k]);
1271		-
1272		- if (offset_in_page(q) + template[i].tap[k] < PAGE_SIZE)
1273		- q[template[i].tap[k]] = 0;
1274		-
1275		- sg_set_buf(&sg[k], q, template[i].tap[k]);
1276		- if (diff_dst) {
1277		- q = xoutbuf[IDX[k] >> PAGE_SHIFT] +
1278		- offset_in_page(IDX[k]);
1279		-
1280		- sg_set_buf(&sgout[k], q, template[i].tap[k]);
1281		-
1282		- memset(q, 0, template[i].tap[k]);
1283		- if (offset_in_page(q) +
1284		- template[i].tap[k] < PAGE_SIZE)
1285		- q[template[i].tap[k]] = 0;
1286		- }
1287		-
1288		- temp += template[i].tap[k];
1289		- }
1290		-
1291		- skcipher_request_set_crypt(req, sg, (diff_dst) ? sgout : sg,
1292		- template[i].len, iv);
1293		-
1294		- ret = crypto_wait_req(enc ? crypto_skcipher_encrypt(req) :
1295		- crypto_skcipher_decrypt(req), &wait);
1296		-
1297		- if (ret) {
1298		- pr_err("alg: skcipher%s: %s failed on chunk test %d for %s: ret=%d\n",
1299		- d, e, j, algo, -ret);
1300		- goto out;
1301		- }
1302		-
1303		- temp = 0;
1304		- ret = -EINVAL;
1305		- for (k = 0; k < template[i].np; k++) {
1306		- if (diff_dst)
1307		- q = xoutbuf[IDX[k] >> PAGE_SHIFT] +
1308		- offset_in_page(IDX[k]);
1309		- else
1310		- q = xbuf[IDX[k] >> PAGE_SHIFT] +
1311		- offset_in_page(IDX[k]);
1312		-
1313		- if (memcmp(q, result + temp, template[i].tap[k])) {
1314		- pr_err("alg: skcipher%s: Chunk test %d failed on %s at page %u for %s\n",
1315		- d, j, e, k, algo);
1316		- hexdump(q, template[i].tap[k]);
1317		- goto out;
1318		- }
1319		-
1320		- q += template[i].tap[k];
1321		- for (n = 0; offset_in_page(q + n) && q[n]; n++)
1322		- ;
1323		- if (n) {
1324		- pr_err("alg: skcipher%s: Result buffer corruption in chunk test %d on %s at page %u for %s: %u bytes:\n",
1325		- d, j, e, k, algo, n);
1326		- hexdump(q, n);
1327		- goto out;
1328		- }
1329		- temp += template[i].tap[k];
1330		- }
	2762	+ /* Build the src/dst scatterlists */
	2763	+ input.iov_base = enc ? (void )vec->ptext : (void )vec->ctext;
	2764	+ input.iov_len = vec->len;
	2765	+ err = build_cipher_test_sglists(tsgls, cfg, alignmask,
	2766	+ vec->len, vec->len, &input, 1);
	2767	+ if (err) {
	2768	+ pr_err("alg: skcipher: %s %s: error preparing scatterlists for test vector %s, cfg=\"%s\"\n",
	2769	+ driver, op, vec_name, cfg->name);
	2770	+ return err;
1331	2771	}
1332	2772
1333		- ret = 0;
	2773	+ /* Do the actual encryption or decryption */
	2774	+ testmgr_poison(req->__ctx, crypto_skcipher_reqsize(tfm));
	2775	+ skcipher_request_set_callback(req, req_flags, crypto_req_done, &wait);
	2776	+ skcipher_request_set_crypt(req, tsgls->src.sgl_ptr, tsgls->dst.sgl_ptr,
	2777	+ vec->len, iv);
	2778	+ if (cfg->nosimd)
	2779	+ crypto_disable_simd_for_test();
	2780	+ err = enc ? crypto_skcipher_encrypt(req) : crypto_skcipher_decrypt(req);
	2781	+ if (cfg->nosimd)
	2782	+ crypto_reenable_simd_for_test();
	2783	+ err = crypto_wait_req(err, &wait);
1334	2784
1335		-out:
1336		- skcipher_request_free(req);
1337		- if (diff_dst)
1338		- testmgr_free_buf(xoutbuf);
1339		-out_nooutbuf:
1340		- testmgr_free_buf(xbuf);
1341		-out_nobuf:
1342		- return ret;
1343		-}
	2785	+ /* Check that the algorithm didn't overwrite things it shouldn't have */
	2786	+ if (req->cryptlen != vec->len \|\|
	2787	+ req->iv != iv \|\|
	2788	+ req->src != tsgls->src.sgl_ptr \|\|
	2789	+ req->dst != tsgls->dst.sgl_ptr \|\|
	2790	+ crypto_skcipher_reqtfm(req) != tfm \|\|
	2791	+ req->base.complete != crypto_req_done \|\|
	2792	+ req->base.flags != req_flags \|\|
	2793	+ req->base.data != &wait) {
	2794	+ pr_err("alg: skcipher: %s %s corrupted request struct on test vector %s, cfg=\"%s\"\n",
	2795	+ driver, op, vec_name, cfg->name);
	2796	+ if (req->cryptlen != vec->len)
	2797	+ pr_err("alg: skcipher: changed 'req->cryptlen'\n");
	2798	+ if (req->iv != iv)
	2799	+ pr_err("alg: skcipher: changed 'req->iv'\n");
	2800	+ if (req->src != tsgls->src.sgl_ptr)
	2801	+ pr_err("alg: skcipher: changed 'req->src'\n");
	2802	+ if (req->dst != tsgls->dst.sgl_ptr)
	2803	+ pr_err("alg: skcipher: changed 'req->dst'\n");
	2804	+ if (crypto_skcipher_reqtfm(req) != tfm)
	2805	+ pr_err("alg: skcipher: changed 'req->base.tfm'\n");
	2806	+ if (req->base.complete != crypto_req_done)
	2807	+ pr_err("alg: skcipher: changed 'req->base.complete'\n");
	2808	+ if (req->base.flags != req_flags)
	2809	+ pr_err("alg: skcipher: changed 'req->base.flags'\n");
	2810	+ if (req->base.data != &wait)
	2811	+ pr_err("alg: skcipher: changed 'req->base.data'\n");
	2812	+ return -EINVAL;
	2813	+ }
	2814	+ if (is_test_sglist_corrupted(&tsgls->src)) {
	2815	+ pr_err("alg: skcipher: %s %s corrupted src sgl on test vector %s, cfg=\"%s\"\n",
	2816	+ driver, op, vec_name, cfg->name);
	2817	+ return -EINVAL;
	2818	+ }
	2819	+ if (tsgls->dst.sgl_ptr != tsgls->src.sgl &&
	2820	+ is_test_sglist_corrupted(&tsgls->dst)) {
	2821	+ pr_err("alg: skcipher: %s %s corrupted dst sgl on test vector %s, cfg=\"%s\"\n",
	2822	+ driver, op, vec_name, cfg->name);
	2823	+ return -EINVAL;
	2824	+ }
1344	2825
1345		-static int test_skcipher(struct crypto_skcipher *tfm, int enc,
1346		- const struct cipher_testvec *template,
1347		- unsigned int tcount)
1348		-{
1349		- unsigned int alignmask;
1350		- int ret;
	2826	+ /* Check for success or failure */
	2827	+ if (err) {
	2828	+ if (err == vec->crypt_error)
	2829	+ return 0;
	2830	+ pr_err("alg: skcipher: %s %s failed on test vector %s; expected_error=%d, actual_error=%d, cfg=\"%s\"\n",
	2831	+ driver, op, vec_name, vec->crypt_error, err, cfg->name);
	2832	+ return err;
	2833	+ }
	2834	+ if (vec->crypt_error) {
	2835	+ pr_err("alg: skcipher: %s %s unexpectedly succeeded on test vector %s; expected_error=%d, cfg=\"%s\"\n",
	2836	+ driver, op, vec_name, vec->crypt_error, cfg->name);
	2837	+ return -EINVAL;
	2838	+ }
1351	2839
1352		- /* test 'dst == src' case */
1353		- ret = __test_skcipher(tfm, enc, template, tcount, false, 0);
1354		- if (ret)
1355		- return ret;
	2840	+ /* Check for the correct output (ciphertext or plaintext) */
	2841	+ err = verify_correct_output(&tsgls->dst, enc ? vec->ctext : vec->ptext,
	2842	+ vec->len, 0, true);
	2843	+ if (err == -EOVERFLOW) {
	2844	+ pr_err("alg: skcipher: %s %s overran dst buffer on test vector %s, cfg=\"%s\"\n",
	2845	+ driver, op, vec_name, cfg->name);
	2846	+ return err;
	2847	+ }
	2848	+ if (err) {
	2849	+ pr_err("alg: skcipher: %s %s test failed (wrong result) on test vector %s, cfg=\"%s\"\n",
	2850	+ driver, op, vec_name, cfg->name);
	2851	+ return err;
	2852	+ }
1356	2853
1357		- /* test 'dst != src' case */
1358		- ret = __test_skcipher(tfm, enc, template, tcount, true, 0);
1359		- if (ret)
1360		- return ret;
1361		-
1362		- /* test unaligned buffers, check with one byte offset */
1363		- ret = __test_skcipher(tfm, enc, template, tcount, true, 1);
1364		- if (ret)
1365		- return ret;
1366		-
1367		- alignmask = crypto_tfm_alg_alignmask(&tfm->base);
1368		- if (alignmask) {
1369		- /* Check if alignment mask for tfm is correctly set. */
1370		- ret = __test_skcipher(tfm, enc, template, tcount, true,
1371		- alignmask + 1);
1372		- if (ret)
1373		- return ret;
	2854	+ /* If applicable, check that the algorithm generated the correct IV */
	2855	+ if (vec->iv_out && memcmp(iv, vec->iv_out, ivsize) != 0) {
	2856	+ pr_err("alg: skcipher: %s %s test failed (wrong output IV) on test vector %s, cfg=\"%s\"\n",
	2857	+ driver, op, vec_name, cfg->name);
	2858	+ hexdump(iv, ivsize);
	2859	+ return -EINVAL;
1374	2860	}
1375	2861
1376	2862	return 0;
	2863	+}
	2864	+
	2865	+static int test_skcipher_vec(const char *driver, int enc,
	2866	+ const struct cipher_testvec *vec,
	2867	+ unsigned int vec_num,
	2868	+ struct skcipher_request *req,
	2869	+ struct cipher_test_sglists *tsgls)
	2870	+{
	2871	+ char vec_name[16];
	2872	+ unsigned int i;
	2873	+ int err;
	2874	+
	2875	+ if (fips_enabled && vec->fips_skip)
	2876	+ return 0;
	2877	+
	2878	+ sprintf(vec_name, "%u", vec_num);
	2879	+
	2880	+ for (i = 0; i < ARRAY_SIZE(default_cipher_testvec_configs); i++) {
	2881	+ err = test_skcipher_vec_cfg(driver, enc, vec, vec_name,
	2882	+ &default_cipher_testvec_configs[i],
	2883	+ req, tsgls);
	2884	+ if (err)
	2885	+ return err;
	2886	+ }
	2887	+
	2888	+#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS
	2889	+ if (!noextratests) {
	2890	+ struct testvec_config cfg;
	2891	+ char cfgname[TESTVEC_CONFIG_NAMELEN];
	2892	+
	2893	+ for (i = 0; i < fuzz_iterations; i++) {
	2894	+ generate_random_testvec_config(&cfg, cfgname,
	2895	+ sizeof(cfgname));
	2896	+ err = test_skcipher_vec_cfg(driver, enc, vec, vec_name,
	2897	+ &cfg, req, tsgls);
	2898	+ if (err)
	2899	+ return err;
	2900	+ cond_resched();
	2901	+ }
	2902	+ }
	2903	+#endif
	2904	+ return 0;
	2905	+}
	2906	+
	2907	+#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS
	2908	+/*
	2909	+ * Generate a symmetric cipher test vector from the given implementation.
	2910	+ * Assumes the buffers in 'vec' were already allocated.
	2911	+ */
	2912	+static void generate_random_cipher_testvec(struct skcipher_request *req,
	2913	+ struct cipher_testvec *vec,
	2914	+ unsigned int maxdatasize,
	2915	+ char *name, size_t max_namelen)
	2916	+{
	2917	+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
	2918	+ const unsigned int maxkeysize = crypto_skcipher_max_keysize(tfm);
	2919	+ const unsigned int ivsize = crypto_skcipher_ivsize(tfm);
	2920	+ struct scatterlist src, dst;
	2921	+ u8 iv[MAX_IVLEN];
	2922	+ DECLARE_CRYPTO_WAIT(wait);
	2923	+
	2924	+ /* Key: length in [0, maxkeysize], but usually choose maxkeysize */
	2925	+ vec->klen = maxkeysize;
	2926	+ if (prandom_u32() % 4 == 0)
	2927	+ vec->klen = prandom_u32() % (maxkeysize + 1);
	2928	+ generate_random_bytes((u8 *)vec->key, vec->klen);
	2929	+ vec->setkey_error = crypto_skcipher_setkey(tfm, vec->key, vec->klen);
	2930	+
	2931	+ /* IV */
	2932	+ generate_random_bytes((u8 *)vec->iv, ivsize);
	2933	+
	2934	+ /* Plaintext */
	2935	+ vec->len = generate_random_length(maxdatasize);
	2936	+ generate_random_bytes((u8 *)vec->ptext, vec->len);
	2937	+
	2938	+ /* If the key couldn't be set, no need to continue to encrypt. */
	2939	+ if (vec->setkey_error)
	2940	+ goto done;
	2941	+
	2942	+ /* Ciphertext */
	2943	+ sg_init_one(&src, vec->ptext, vec->len);
	2944	+ sg_init_one(&dst, vec->ctext, vec->len);
	2945	+ memcpy(iv, vec->iv, ivsize);
	2946	+ skcipher_request_set_callback(req, 0, crypto_req_done, &wait);
	2947	+ skcipher_request_set_crypt(req, &src, &dst, vec->len, iv);
	2948	+ vec->crypt_error = crypto_wait_req(crypto_skcipher_encrypt(req), &wait);
	2949	+ if (vec->crypt_error != 0) {
	2950	+ /*
	2951	+ * The only acceptable error here is for an invalid length, so
	2952	+ * skcipher decryption should fail with the same error too.
	2953	+ * We'll test for this. But to keep the API usage well-defined,
	2954	+ * explicitly initialize the ciphertext buffer too.
	2955	+ */
	2956	+ memset((u8 *)vec->ctext, 0, vec->len);
	2957	+ }
	2958	+done:
	2959	+ snprintf(name, max_namelen, "\"random: len=%u klen=%u\"",
	2960	+ vec->len, vec->klen);
	2961	+}
	2962	+
	2963	+/*
	2964	+ * Test the skcipher algorithm represented by @req against the corresponding
	2965	+ * generic implementation, if one is available.
	2966	+ */
	2967	+static int test_skcipher_vs_generic_impl(const char *driver,
	2968	+ const char *generic_driver,
	2969	+ struct skcipher_request *req,
	2970	+ struct cipher_test_sglists *tsgls)
	2971	+{
	2972	+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
	2973	+ const unsigned int maxkeysize = crypto_skcipher_max_keysize(tfm);
	2974	+ const unsigned int ivsize = crypto_skcipher_ivsize(tfm);
	2975	+ const unsigned int blocksize = crypto_skcipher_blocksize(tfm);
	2976	+ const unsigned int maxdatasize = (2 * PAGE_SIZE) - TESTMGR_POISON_LEN;
	2977	+ const char *algname = crypto_skcipher_alg(tfm)->base.cra_name;
	2978	+ char _generic_driver[CRYPTO_MAX_ALG_NAME];
	2979	+ struct crypto_skcipher *generic_tfm = NULL;
	2980	+ struct skcipher_request *generic_req = NULL;
	2981	+ unsigned int i;
	2982	+ struct cipher_testvec vec = { 0 };
	2983	+ char vec_name[64];
	2984	+ struct testvec_config *cfg;
	2985	+ char cfgname[TESTVEC_CONFIG_NAMELEN];
	2986	+ int err;
	2987	+
	2988	+ if (noextratests)
	2989	+ return 0;
	2990	+
	2991	+ /* Keywrap isn't supported here yet as it handles its IV differently. */
	2992	+ if (strncmp(algname, "kw(", 3) == 0)
	2993	+ return 0;
	2994	+
	2995	+ if (!generic_driver) { /* Use default naming convention? */
	2996	+ err = build_generic_driver_name(algname, _generic_driver);
	2997	+ if (err)
	2998	+ return err;
	2999	+ generic_driver = _generic_driver;
	3000	+ }
	3001	+
	3002	+ if (strcmp(generic_driver, driver) == 0) /* Already the generic impl? */
	3003	+ return 0;
	3004	+
	3005	+ generic_tfm = crypto_alloc_skcipher(generic_driver, 0, 0);
	3006	+ if (IS_ERR(generic_tfm)) {
	3007	+ err = PTR_ERR(generic_tfm);
	3008	+ if (err == -ENOENT) {
	3009	+ pr_warn("alg: skcipher: skipping comparison tests for %s because %s is unavailable\n",
	3010	+ driver, generic_driver);
	3011	+ return 0;
	3012	+ }
	3013	+ pr_err("alg: skcipher: error allocating %s (generic impl of %s): %d\n",
	3014	+ generic_driver, algname, err);
	3015	+ return err;
	3016	+ }
	3017	+
	3018	+ cfg = kzalloc(sizeof(*cfg), GFP_KERNEL);
	3019	+ if (!cfg) {
	3020	+ err = -ENOMEM;
	3021	+ goto out;
	3022	+ }
	3023	+
	3024	+ generic_req = skcipher_request_alloc(generic_tfm, GFP_KERNEL);
	3025	+ if (!generic_req) {
	3026	+ err = -ENOMEM;
	3027	+ goto out;
	3028	+ }
	3029	+
	3030	+ /* Check the algorithm properties for consistency. */
	3031	+
	3032	+ if (crypto_skcipher_min_keysize(tfm) !=
	3033	+ crypto_skcipher_min_keysize(generic_tfm)) {
	3034	+ pr_err("alg: skcipher: min keysize for %s (%u) doesn't match generic impl (%u)\n",
	3035	+ driver, crypto_skcipher_min_keysize(tfm),
	3036	+ crypto_skcipher_min_keysize(generic_tfm));
	3037	+ err = -EINVAL;
	3038	+ goto out;
	3039	+ }
	3040	+
	3041	+ if (maxkeysize != crypto_skcipher_max_keysize(generic_tfm)) {
	3042	+ pr_err("alg: skcipher: max keysize for %s (%u) doesn't match generic impl (%u)\n",
	3043	+ driver, maxkeysize,
	3044	+ crypto_skcipher_max_keysize(generic_tfm));
	3045	+ err = -EINVAL;
	3046	+ goto out;
	3047	+ }
	3048	+
	3049	+ if (ivsize != crypto_skcipher_ivsize(generic_tfm)) {
	3050	+ pr_err("alg: skcipher: ivsize for %s (%u) doesn't match generic impl (%u)\n",
	3051	+ driver, ivsize, crypto_skcipher_ivsize(generic_tfm));
	3052	+ err = -EINVAL;
	3053	+ goto out;
	3054	+ }
	3055	+
	3056	+ if (blocksize != crypto_skcipher_blocksize(generic_tfm)) {
	3057	+ pr_err("alg: skcipher: blocksize for %s (%u) doesn't match generic impl (%u)\n",
	3058	+ driver, blocksize,
	3059	+ crypto_skcipher_blocksize(generic_tfm));
	3060	+ err = -EINVAL;
	3061	+ goto out;
	3062	+ }
	3063	+
	3064	+ /*
	3065	+ * Now generate test vectors using the generic implementation, and test
	3066	+ * the other implementation against them.
	3067	+ */
	3068	+
	3069	+ vec.key = kmalloc(maxkeysize, GFP_KERNEL);
	3070	+ vec.iv = kmalloc(ivsize, GFP_KERNEL);
	3071	+ vec.ptext = kmalloc(maxdatasize, GFP_KERNEL);
	3072	+ vec.ctext = kmalloc(maxdatasize, GFP_KERNEL);
	3073	+ if (!vec.key \|\| !vec.iv \|\| !vec.ptext \|\| !vec.ctext) {
	3074	+ err = -ENOMEM;
	3075	+ goto out;
	3076	+ }
	3077	+
	3078	+ for (i = 0; i < fuzz_iterations * 8; i++) {
	3079	+ generate_random_cipher_testvec(generic_req, &vec, maxdatasize,
	3080	+ vec_name, sizeof(vec_name));
	3081	+ generate_random_testvec_config(cfg, cfgname, sizeof(cfgname));
	3082	+
	3083	+ err = test_skcipher_vec_cfg(driver, ENCRYPT, &vec, vec_name,
	3084	+ cfg, req, tsgls);
	3085	+ if (err)
	3086	+ goto out;
	3087	+ err = test_skcipher_vec_cfg(driver, DECRYPT, &vec, vec_name,
	3088	+ cfg, req, tsgls);
	3089	+ if (err)
	3090	+ goto out;
	3091	+ cond_resched();
	3092	+ }
	3093	+ err = 0;
	3094	+out:
	3095	+ kfree(cfg);
	3096	+ kfree(vec.key);
	3097	+ kfree(vec.iv);
	3098	+ kfree(vec.ptext);
	3099	+ kfree(vec.ctext);
	3100	+ crypto_free_skcipher(generic_tfm);
	3101	+ skcipher_request_free(generic_req);
	3102	+ return err;
	3103	+}
	3104	+#else /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */
	3105	+static int test_skcipher_vs_generic_impl(const char *driver,
	3106	+ const char *generic_driver,
	3107	+ struct skcipher_request *req,
	3108	+ struct cipher_test_sglists *tsgls)
	3109	+{
	3110	+ return 0;
	3111	+}
	3112	+#endif /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */
	3113	+
	3114	+static int test_skcipher(const char *driver, int enc,
	3115	+ const struct cipher_test_suite *suite,
	3116	+ struct skcipher_request *req,
	3117	+ struct cipher_test_sglists *tsgls)
	3118	+{
	3119	+ unsigned int i;
	3120	+ int err;
	3121	+
	3122	+ for (i = 0; i < suite->count; i++) {
	3123	+ err = test_skcipher_vec(driver, enc, &suite->vecs[i], i, req,
	3124	+ tsgls);
	3125	+ if (err)
	3126	+ return err;
	3127	+ cond_resched();
	3128	+ }
	3129	+ return 0;
	3130	+}
	3131	+
	3132	+static int alg_test_skcipher(const struct alg_test_desc *desc,
	3133	+ const char *driver, u32 type, u32 mask)
	3134	+{
	3135	+ const struct cipher_test_suite *suite = &desc->suite.cipher;
	3136	+ struct crypto_skcipher *tfm;
	3137	+ struct skcipher_request *req = NULL;
	3138	+ struct cipher_test_sglists *tsgls = NULL;
	3139	+ int err;
	3140	+
	3141	+ if (suite->count <= 0) {
	3142	+ pr_err("alg: skcipher: empty test suite for %s\n", driver);
	3143	+ return -EINVAL;
	3144	+ }
	3145	+
	3146	+ tfm = crypto_alloc_skcipher(driver, type, mask);
	3147	+ if (IS_ERR(tfm)) {
	3148	+ pr_err("alg: skcipher: failed to allocate transform for %s: %ld\n",
	3149	+ driver, PTR_ERR(tfm));
	3150	+ return PTR_ERR(tfm);
	3151	+ }
	3152	+
	3153	+ req = skcipher_request_alloc(tfm, GFP_KERNEL);
	3154	+ if (!req) {
	3155	+ pr_err("alg: skcipher: failed to allocate request for %s\n",
	3156	+ driver);
	3157	+ err = -ENOMEM;
	3158	+ goto out;
	3159	+ }
	3160	+
	3161	+ tsgls = alloc_cipher_test_sglists();
	3162	+ if (!tsgls) {
	3163	+ pr_err("alg: skcipher: failed to allocate test buffers for %s\n",
	3164	+ driver);
	3165	+ err = -ENOMEM;
	3166	+ goto out;
	3167	+ }
	3168	+
	3169	+ err = test_skcipher(driver, ENCRYPT, suite, req, tsgls);
	3170	+ if (err)
	3171	+ goto out;
	3172	+
	3173	+ err = test_skcipher(driver, DECRYPT, suite, req, tsgls);
	3174	+ if (err)
	3175	+ goto out;
	3176	+
	3177	+ err = test_skcipher_vs_generic_impl(driver, desc->generic_driver, req,
	3178	+ tsgls);
	3179	+out:
	3180	+ free_cipher_test_sglists(tsgls);
	3181	+ skcipher_request_free(req);
	3182	+ crypto_free_skcipher(tfm);
	3183	+ return err;
1377	3184	}
1378	3185
1379	3186	static int test_comp(struct crypto_comp *tfm,
..	..	@@ -1713,35 +3520,6 @@
1713	3520	return err;
1714	3521	}
1715	3522
1716		-static int alg_test_aead(const struct alg_test_desc desc, const char driver,
1717		- u32 type, u32 mask)
1718		-{
1719		- struct crypto_aead *tfm;
1720		- int err = 0;
1721		-
1722		- tfm = crypto_alloc_aead(driver, type, mask);
1723		- if (IS_ERR(tfm)) {
1724		- printk(KERN_ERR "alg: aead: Failed to load transform for %s: "
1725		- "%ld\n", driver, PTR_ERR(tfm));
1726		- return PTR_ERR(tfm);
1727		- }
1728		-
1729		- if (desc->suite.aead.enc.vecs) {
1730		- err = test_aead(tfm, ENCRYPT, desc->suite.aead.enc.vecs,
1731		- desc->suite.aead.enc.count);
1732		- if (err)
1733		- goto out;
1734		- }
1735		-
1736		- if (!err && desc->suite.aead.dec.vecs)
1737		- err = test_aead(tfm, DECRYPT, desc->suite.aead.dec.vecs,
1738		- desc->suite.aead.dec.count);
1739		-
1740		-out:
1741		- crypto_free_aead(tfm);
1742		- return err;
1743		-}
1744		-
1745	3523	static int alg_test_cipher(const struct alg_test_desc *desc,
1746	3524	const char *driver, u32 type, u32 mask)
1747	3525	{
..	..	@@ -1761,28 +3539,6 @@
1761	3539	err = test_cipher(tfm, DECRYPT, suite->vecs, suite->count);
1762	3540
1763	3541	crypto_free_cipher(tfm);
1764		- return err;
1765		-}
1766		-
1767		-static int alg_test_skcipher(const struct alg_test_desc *desc,
1768		- const char *driver, u32 type, u32 mask)
1769		-{
1770		- const struct cipher_test_suite *suite = &desc->suite.cipher;
1771		- struct crypto_skcipher *tfm;
1772		- int err;
1773		-
1774		- tfm = crypto_alloc_skcipher(driver, type, mask);
1775		- if (IS_ERR(tfm)) {
1776		- printk(KERN_ERR "alg: skcipher: Failed to load transform for "
1777		- "%s: %ld\n", driver, PTR_ERR(tfm));
1778		- return PTR_ERR(tfm);
1779		- }
1780		-
1781		- err = test_skcipher(tfm, ENCRYPT, suite->vecs, suite->count);
1782		- if (!err)
1783		- err = test_skcipher(tfm, DECRYPT, suite->vecs, suite->count);
1784		-
1785		- crypto_free_skcipher(tfm);
1786	3542	return err;
1787	3543	}
1788	3544
..	..	@@ -1824,72 +3580,11 @@
1824	3580	return err;
1825	3581	}
1826	3582
1827		-static int __alg_test_hash(const struct hash_testvec *template,
1828		- unsigned int tcount, const char *driver,
1829		- u32 type, u32 mask)
1830		-{
1831		- struct crypto_ahash *tfm;
1832		- int err;
1833		-
1834		- tfm = crypto_alloc_ahash(driver, type, mask);
1835		- if (IS_ERR(tfm)) {
1836		- printk(KERN_ERR "alg: hash: Failed to load transform for %s: "
1837		- "%ld\n", driver, PTR_ERR(tfm));
1838		- return PTR_ERR(tfm);
1839		- }
1840		-
1841		- err = test_hash(tfm, template, tcount, HASH_TEST_DIGEST);
1842		- if (!err)
1843		- err = test_hash(tfm, template, tcount, HASH_TEST_FINAL);
1844		- if (!err)
1845		- err = test_hash(tfm, template, tcount, HASH_TEST_FINUP);
1846		- crypto_free_ahash(tfm);
1847		- return err;
1848		-}
1849		-
1850		-static int alg_test_hash(const struct alg_test_desc desc, const char driver,
1851		- u32 type, u32 mask)
1852		-{
1853		- const struct hash_testvec *template = desc->suite.hash.vecs;
1854		- unsigned int tcount = desc->suite.hash.count;
1855		- unsigned int nr_unkeyed, nr_keyed;
1856		- int err;
1857		-
1858		- /*
1859		- * For OPTIONAL_KEY algorithms, we have to do all the unkeyed tests
1860		- * first, before setting a key on the tfm. To make this easier, we
1861		- * require that the unkeyed test vectors (if any) are listed first.
1862		- */
1863		-
1864		- for (nr_unkeyed = 0; nr_unkeyed < tcount; nr_unkeyed++) {
1865		- if (template[nr_unkeyed].ksize)
1866		- break;
1867		- }
1868		- for (nr_keyed = 0; nr_unkeyed + nr_keyed < tcount; nr_keyed++) {
1869		- if (!template[nr_unkeyed + nr_keyed].ksize) {
1870		- pr_err("alg: hash: test vectors for %s out of order, "
1871		- "unkeyed ones must come first\n", desc->alg);
1872		- return -EINVAL;
1873		- }
1874		- }
1875		-
1876		- err = 0;
1877		- if (nr_unkeyed) {
1878		- err = __alg_test_hash(template, nr_unkeyed, driver, type, mask);
1879		- template += nr_unkeyed;
1880		- }
1881		-
1882		- if (!err && nr_keyed)
1883		- err = __alg_test_hash(template, nr_keyed, driver, type, mask);
1884		-
1885		- return err;
1886		-}
1887		-
1888	3583	static int alg_test_crc32c(const struct alg_test_desc *desc,
1889	3584	const char *driver, u32 type, u32 mask)
1890	3585	{
1891	3586	struct crypto_shash *tfm;
1892		- u32 val;
	3587	+ __le32 val;
1893	3588	int err;
1894	3589
1895	3590	err = alg_test_hash(desc, driver, type, mask);
..	..	@@ -1916,9 +3611,8 @@
1916	3611	u32 ctx = (u32 )shash_desc_ctx(shash);
1917	3612
1918	3613	shash->tfm = tfm;
1919		- shash->flags = 0;
1920	3614
1921		- *ctx = le32_to_cpu(420553207);
	3615	+ *ctx = 420553207;
1922	3616	err = crypto_shash_final(shash, (u8 *)&val);
1923	3617	if (err) {
1924	3618	printk(KERN_ERR "alg: crc32c: Operation failed for "
..	..	@@ -1926,9 +3620,9 @@
1926	3620	break;
1927	3621	}
1928	3622
1929		- if (val != ~420553207) {
1930		- printk(KERN_ERR "alg: crc32c: Test failed for %s: "
1931		- "%d\n", driver, val);
	3623	+ if (val != cpu_to_le32(~420553207)) {
	3624	+ pr_err("alg: crc32c: Test failed for %s: %u\n",
	3625	+ driver, le32_to_cpu(val));
1932	3626	err = -EINVAL;
1933	3627	}
1934	3628	} while (0);
..	..	@@ -1975,7 +3669,7 @@
1975	3669	if (IS_ERR(drng)) {
1976	3670	printk(KERN_ERR "alg: drbg: could not allocate DRNG handle for "
1977	3671	"%s\n", driver);
1978		- kzfree(buf);
	3672	+ kfree_sensitive(buf);
1979	3673	return -ENOMEM;
1980	3674	}
1981	3675
..	..	@@ -2022,7 +3716,7 @@
2022	3716
2023	3717	outbuf:
2024	3718	crypto_free_rng(drng);
2025		- kzfree(buf);
	3719	+ kfree_sensitive(buf);
2026	3720	return ret;
2027	3721	}
2028	3722
..	..	@@ -2100,12 +3794,11 @@
2100	3794
2101	3795	if (vec->genkey) {
2102	3796	/* Save party A's public key */
2103		- a_public = kzalloc(out_len_max, GFP_KERNEL);
	3797	+ a_public = kmemdup(sg_virt(req->dst), out_len_max, GFP_KERNEL);
2104	3798	if (!a_public) {
2105	3799	err = -ENOMEM;
2106	3800	goto free_output;
2107	3801	}
2108		- memcpy(a_public, sg_virt(req->dst), out_len_max);
2109	3802	} else {
2110	3803	/* Verify calculated public key */
2111	3804	if (memcmp(vec->expected_a_public, sg_virt(req->dst),
..	..	@@ -2118,13 +3811,12 @@
2118	3811	}
2119	3812
2120	3813	/* Calculate shared secret key by using counter part (b) public key. */
2121		- input_buf = kzalloc(vec->b_public_size, GFP_KERNEL);
	3814	+ input_buf = kmemdup(vec->b_public, vec->b_public_size, GFP_KERNEL);
2122	3815	if (!input_buf) {
2123	3816	err = -ENOMEM;
2124	3817	goto free_output;
2125	3818	}
2126	3819
2127		- memcpy(input_buf, vec->b_public, vec->b_public_size);
2128	3820	sg_init_one(&src, input_buf, vec->b_public_size);
2129	3821	sg_init_one(&dst, output_buf, out_len_max);
2130	3822	kpp_request_set_input(req, &src, vec->b_public_size);
..	..	@@ -2140,12 +3832,11 @@
2140	3832
2141	3833	if (vec->genkey) {
2142	3834	/* Save the shared secret obtained by party A */
2143		- a_ss = kzalloc(vec->expected_ss_size, GFP_KERNEL);
	3835	+ a_ss = kmemdup(sg_virt(req->dst), vec->expected_ss_size, GFP_KERNEL);
2144	3836	if (!a_ss) {
2145	3837	err = -ENOMEM;
2146	3838	goto free_all;
2147	3839	}
2148		- memcpy(a_ss, sg_virt(req->dst), vec->expected_ss_size);
2149	3840
2150	3841	/*
2151	3842	* Calculate party B's shared secret by using party A's
..	..	@@ -2233,6 +3924,12 @@
2233	3924	return err;
2234	3925	}
2235	3926
	3927	+static u8 test_pack_u32(u8 dst, u32 val)
	3928	+{
	3929	+ memcpy(dst, &val, sizeof(val));
	3930	+ return dst + sizeof(val);
	3931	+}
	3932	+
2236	3933	static int test_akcipher_one(struct crypto_akcipher *tfm,
2237	3934	const struct akcipher_testvec *vecs)
2238	3935	{
..	..	@@ -2243,7 +3940,11 @@
2243	3940	struct crypto_wait wait;
2244	3941	unsigned int out_len_max, out_len = 0;
2245	3942	int err = -ENOMEM;
2246		- struct scatterlist src, dst, src_tab[2];
	3943	+ struct scatterlist src, dst, src_tab[3];
	3944	+ const char m, c;
	3945	+ unsigned int m_size, c_size;
	3946	+ const char *op;
	3947	+ u8 key, ptr;
2247	3948
2248	3949	if (testmgr_alloc_buf(xbuf))
2249	3950	return err;
..	..	@@ -2254,57 +3955,102 @@
2254	3955
2255	3956	crypto_init_wait(&wait);
2256	3957
2257		- if (vecs->public_key_vec)
2258		- err = crypto_akcipher_set_pub_key(tfm, vecs->key,
2259		- vecs->key_len);
2260		- else
2261		- err = crypto_akcipher_set_priv_key(tfm, vecs->key,
2262		- vecs->key_len);
2263		- if (err)
	3958	+ key = kmalloc(vecs->key_len + sizeof(u32) * 2 + vecs->param_len,
	3959	+ GFP_KERNEL);
	3960	+ if (!key)
2264	3961	goto free_req;
	3962	+ memcpy(key, vecs->key, vecs->key_len);
	3963	+ ptr = key + vecs->key_len;
	3964	+ ptr = test_pack_u32(ptr, vecs->algo);
	3965	+ ptr = test_pack_u32(ptr, vecs->param_len);
	3966	+ memcpy(ptr, vecs->params, vecs->param_len);
2265	3967
	3968	+ if (vecs->public_key_vec)
	3969	+ err = crypto_akcipher_set_pub_key(tfm, key, vecs->key_len);
	3970	+ else
	3971	+ err = crypto_akcipher_set_priv_key(tfm, key, vecs->key_len);
	3972	+ if (err)
	3973	+ goto free_key;
	3974	+
	3975	+ /*
	3976	+ * First run test which do not require a private key, such as
	3977	+ * encrypt or verify.
	3978	+ */
2266	3979	err = -ENOMEM;
2267	3980	out_len_max = crypto_akcipher_maxsize(tfm);
2268	3981	outbuf_enc = kzalloc(out_len_max, GFP_KERNEL);
2269	3982	if (!outbuf_enc)
2270		- goto free_req;
	3983	+ goto free_key;
2271	3984
2272		- if (WARN_ON(vecs->m_size > PAGE_SIZE))
	3985	+ if (!vecs->siggen_sigver_test) {
	3986	+ m = vecs->m;
	3987	+ m_size = vecs->m_size;
	3988	+ c = vecs->c;
	3989	+ c_size = vecs->c_size;
	3990	+ op = "encrypt";
	3991	+ } else {
	3992	+ /* Swap args so we could keep plaintext (digest)
	3993	+ * in vecs->m, and cooked signature in vecs->c.
	3994	+ */
	3995	+ m = vecs->c; /* signature */
	3996	+ m_size = vecs->c_size;
	3997	+ c = vecs->m; /* digest */
	3998	+ c_size = vecs->m_size;
	3999	+ op = "verify";
	4000	+ }
	4001	+
	4002	+ err = -E2BIG;
	4003	+ if (WARN_ON(m_size > PAGE_SIZE))
2273	4004	goto free_all;
	4005	+ memcpy(xbuf[0], m, m_size);
2274	4006
2275		- memcpy(xbuf[0], vecs->m, vecs->m_size);
2276		-
2277		- sg_init_table(src_tab, 2);
	4007	+ sg_init_table(src_tab, 3);
2278	4008	sg_set_buf(&src_tab[0], xbuf[0], 8);
2279		- sg_set_buf(&src_tab[1], xbuf[0] + 8, vecs->m_size - 8);
2280		- sg_init_one(&dst, outbuf_enc, out_len_max);
2281		- akcipher_request_set_crypt(req, src_tab, &dst, vecs->m_size,
2282		- out_len_max);
	4009	+ sg_set_buf(&src_tab[1], xbuf[0] + 8, m_size - 8);
	4010	+ if (vecs->siggen_sigver_test) {
	4011	+ if (WARN_ON(c_size > PAGE_SIZE))
	4012	+ goto free_all;
	4013	+ memcpy(xbuf[1], c, c_size);
	4014	+ sg_set_buf(&src_tab[2], xbuf[1], c_size);
	4015	+ akcipher_request_set_crypt(req, src_tab, NULL, m_size, c_size);
	4016	+ } else {
	4017	+ sg_init_one(&dst, outbuf_enc, out_len_max);
	4018	+ akcipher_request_set_crypt(req, src_tab, &dst, m_size,
	4019	+ out_len_max);
	4020	+ }
2283	4021	akcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
2284	4022	crypto_req_done, &wait);
2285	4023
2286	4024	err = crypto_wait_req(vecs->siggen_sigver_test ?
2287		- /* Run asymmetric signature generation */
2288		- crypto_akcipher_sign(req) :
	4025	+ /* Run asymmetric signature verification */
	4026	+ crypto_akcipher_verify(req) :
2289	4027	/* Run asymmetric encrypt */
2290	4028	crypto_akcipher_encrypt(req), &wait);
2291	4029	if (err) {
2292		- pr_err("alg: akcipher: encrypt test failed. err %d\n", err);
	4030	+ pr_err("alg: akcipher: %s test failed. err %d\n", op, err);
2293	4031	goto free_all;
2294	4032	}
2295		- if (req->dst_len != vecs->c_size) {
2296		- pr_err("alg: akcipher: encrypt test failed. Invalid output len\n");
2297		- err = -EINVAL;
2298		- goto free_all;
	4033	+ if (!vecs->siggen_sigver_test && c) {
	4034	+ if (req->dst_len != c_size) {
	4035	+ pr_err("alg: akcipher: %s test failed. Invalid output len\n",
	4036	+ op);
	4037	+ err = -EINVAL;
	4038	+ goto free_all;
	4039	+ }
	4040	+ /* verify that encrypted message is equal to expected */
	4041	+ if (memcmp(c, outbuf_enc, c_size) != 0) {
	4042	+ pr_err("alg: akcipher: %s test failed. Invalid output\n",
	4043	+ op);
	4044	+ hexdump(outbuf_enc, c_size);
	4045	+ err = -EINVAL;
	4046	+ goto free_all;
	4047	+ }
2299	4048	}
2300		- /* verify that encrypted message is equal to expected */
2301		- if (memcmp(vecs->c, outbuf_enc, vecs->c_size)) {
2302		- pr_err("alg: akcipher: encrypt test failed. Invalid output\n");
2303		- hexdump(outbuf_enc, vecs->c_size);
2304		- err = -EINVAL;
2305		- goto free_all;
2306		- }
2307		- /* Don't invoke decrypt for vectors with public key */
	4049	+
	4050	+ /*
	4051	+ * Don't invoke (decrypt or sign) test which require a private key
	4052	+ * for vectors with only a public key.
	4053	+ */
2308	4054	if (vecs->public_key_vec) {
2309	4055	err = 0;
2310	4056	goto free_all;
..	..	@@ -2315,43 +4061,50 @@
2315	4061	goto free_all;
2316	4062	}
2317	4063
2318		- if (WARN_ON(vecs->c_size > PAGE_SIZE))
	4064	+ if (!vecs->siggen_sigver_test && !c) {
	4065	+ c = outbuf_enc;
	4066	+ c_size = req->dst_len;
	4067	+ }
	4068	+
	4069	+ err = -E2BIG;
	4070	+ op = vecs->siggen_sigver_test ? "sign" : "decrypt";
	4071	+ if (WARN_ON(c_size > PAGE_SIZE))
2319	4072	goto free_all;
	4073	+ memcpy(xbuf[0], c, c_size);
2320	4074
2321		- memcpy(xbuf[0], vecs->c, vecs->c_size);
2322		-
2323		- sg_init_one(&src, xbuf[0], vecs->c_size);
	4075	+ sg_init_one(&src, xbuf[0], c_size);
2324	4076	sg_init_one(&dst, outbuf_dec, out_len_max);
2325	4077	crypto_init_wait(&wait);
2326		- akcipher_request_set_crypt(req, &src, &dst, vecs->c_size, out_len_max);
	4078	+ akcipher_request_set_crypt(req, &src, &dst, c_size, out_len_max);
2327	4079
2328	4080	err = crypto_wait_req(vecs->siggen_sigver_test ?
2329		- /* Run asymmetric signature verification */
2330		- crypto_akcipher_verify(req) :
	4081	+ /* Run asymmetric signature generation */
	4082	+ crypto_akcipher_sign(req) :
2331	4083	/* Run asymmetric decrypt */
2332	4084	crypto_akcipher_decrypt(req), &wait);
2333	4085	if (err) {
2334		- pr_err("alg: akcipher: decrypt test failed. err %d\n", err);
	4086	+ pr_err("alg: akcipher: %s test failed. err %d\n", op, err);
2335	4087	goto free_all;
2336	4088	}
2337	4089	out_len = req->dst_len;
2338		- if (out_len < vecs->m_size) {
2339		- pr_err("alg: akcipher: decrypt test failed. "
2340		- "Invalid output len %u\n", out_len);
	4090	+ if (out_len < m_size) {
	4091	+ pr_err("alg: akcipher: %s test failed. Invalid output len %u\n",
	4092	+ op, out_len);
2341	4093	err = -EINVAL;
2342	4094	goto free_all;
2343	4095	}
2344	4096	/* verify that decrypted message is equal to the original msg */
2345		- if (memchr_inv(outbuf_dec, 0, out_len - vecs->m_size) \|\|
2346		- memcmp(vecs->m, outbuf_dec + out_len - vecs->m_size,
2347		- vecs->m_size)) {
2348		- pr_err("alg: akcipher: decrypt test failed. Invalid output\n");
	4097	+ if (memchr_inv(outbuf_dec, 0, out_len - m_size) \|\|
	4098	+ memcmp(m, outbuf_dec + out_len - m_size, m_size)) {
	4099	+ pr_err("alg: akcipher: %s test failed. Invalid output\n", op);
2349	4100	hexdump(outbuf_dec, out_len);
2350	4101	err = -EINVAL;
2351	4102	}
2352	4103	free_all:
2353	4104	kfree(outbuf_dec);
2354	4105	kfree(outbuf_enc);
	4106	+free_key:
	4107	+ kfree(key);
2355	4108	free_req:
2356	4109	akcipher_request_free(req);
2357	4110	free_xbuf:
..	..	@@ -2405,18 +4158,21 @@
2405	4158	return 0;
2406	4159	}
2407	4160
2408		-#define __VECS(tv) { .vecs = tv, .count = ARRAY_SIZE(tv) }
	4161	+#define ____VECS(tv) .vecs = tv, .count = ARRAY_SIZE(tv)
	4162	+#define __VECS(tv) { ____VECS(tv) }
2409	4163
2410	4164	/* Please keep this list sorted by algorithm name. */
2411	4165	static const struct alg_test_desc alg_test_descs[] = {
2412	4166	{
2413	4167	.alg = "adiantum(xchacha12,aes)",
	4168	+ .generic_driver = "adiantum(xchacha12-generic,aes-generic,nhpoly1305-generic)",
2414	4169	.test = alg_test_skcipher,
2415	4170	.suite = {
2416	4171	.cipher = __VECS(adiantum_xchacha12_aes_tv_template)
2417	4172	},
2418	4173	}, {
2419	4174	.alg = "adiantum(xchacha20,aes)",
	4175	+ .generic_driver = "adiantum(xchacha20-generic,aes-generic,nhpoly1305-generic)",
2420	4176	.test = alg_test_skcipher,
2421	4177	.suite = {
2422	4178	.cipher = __VECS(adiantum_xchacha20_aes_tv_template)
..	..	@@ -2425,28 +4181,7 @@
2425	4181	.alg = "aegis128",
2426	4182	.test = alg_test_aead,
2427	4183	.suite = {
2428		- .aead = {
2429		- .enc = __VECS(aegis128_enc_tv_template),
2430		- .dec = __VECS(aegis128_dec_tv_template),
2431		- }
2432		- }
2433		- }, {
2434		- .alg = "aegis128l",
2435		- .test = alg_test_aead,
2436		- .suite = {
2437		- .aead = {
2438		- .enc = __VECS(aegis128l_enc_tv_template),
2439		- .dec = __VECS(aegis128l_dec_tv_template),
2440		- }
2441		- }
2442		- }, {
2443		- .alg = "aegis256",
2444		- .test = alg_test_aead,
2445		- .suite = {
2446		- .aead = {
2447		- .enc = __VECS(aegis256_enc_tv_template),
2448		- .dec = __VECS(aegis256_dec_tv_template),
2449		- }
	4184	+ .aead = __VECS(aegis128_tv_template)
2450	4185	}
2451	4186	}, {
2452	4187	.alg = "ansi_cprng",
..	..	@@ -2458,36 +4193,27 @@
2458	4193	.alg = "authenc(hmac(md5),ecb(cipher_null))",
2459	4194	.test = alg_test_aead,
2460	4195	.suite = {
2461		- .aead = {
2462		- .enc = __VECS(hmac_md5_ecb_cipher_null_enc_tv_template),
2463		- .dec = __VECS(hmac_md5_ecb_cipher_null_dec_tv_template)
2464		- }
	4196	+ .aead = __VECS(hmac_md5_ecb_cipher_null_tv_template)
2465	4197	}
2466	4198	}, {
2467	4199	.alg = "authenc(hmac(sha1),cbc(aes))",
2468	4200	.test = alg_test_aead,
2469	4201	.fips_allowed = 1,
2470	4202	.suite = {
2471		- .aead = {
2472		- .enc = __VECS(hmac_sha1_aes_cbc_enc_tv_temp)
2473		- }
	4203	+ .aead = __VECS(hmac_sha1_aes_cbc_tv_temp)
2474	4204	}
2475	4205	}, {
2476	4206	.alg = "authenc(hmac(sha1),cbc(des))",
2477	4207	.test = alg_test_aead,
2478	4208	.suite = {
2479		- .aead = {
2480		- .enc = __VECS(hmac_sha1_des_cbc_enc_tv_temp)
2481		- }
	4209	+ .aead = __VECS(hmac_sha1_des_cbc_tv_temp)
2482	4210	}
2483	4211	}, {
2484	4212	.alg = "authenc(hmac(sha1),cbc(des3_ede))",
2485	4213	.test = alg_test_aead,
2486	4214	.fips_allowed = 1,
2487	4215	.suite = {
2488		- .aead = {
2489		- .enc = __VECS(hmac_sha1_des3_ede_cbc_enc_tv_temp)
2490		- }
	4216	+ .aead = __VECS(hmac_sha1_des3_ede_cbc_tv_temp)
2491	4217	}
2492	4218	}, {
2493	4219	.alg = "authenc(hmac(sha1),ctr(aes))",
..	..	@@ -2497,10 +4223,7 @@
2497	4223	.alg = "authenc(hmac(sha1),ecb(cipher_null))",
2498	4224	.test = alg_test_aead,
2499	4225	.suite = {
2500		- .aead = {
2501		- .enc = __VECS(hmac_sha1_ecb_cipher_null_enc_tv_temp),
2502		- .dec = __VECS(hmac_sha1_ecb_cipher_null_dec_tv_temp)
2503		- }
	4226	+ .aead = __VECS(hmac_sha1_ecb_cipher_null_tv_temp)
2504	4227	}
2505	4228	}, {
2506	4229	.alg = "authenc(hmac(sha1),rfc3686(ctr(aes)))",
..	..	@@ -2510,44 +4233,34 @@
2510	4233	.alg = "authenc(hmac(sha224),cbc(des))",
2511	4234	.test = alg_test_aead,
2512	4235	.suite = {
2513		- .aead = {
2514		- .enc = __VECS(hmac_sha224_des_cbc_enc_tv_temp)
2515		- }
	4236	+ .aead = __VECS(hmac_sha224_des_cbc_tv_temp)
2516	4237	}
2517	4238	}, {
2518	4239	.alg = "authenc(hmac(sha224),cbc(des3_ede))",
2519	4240	.test = alg_test_aead,
2520	4241	.fips_allowed = 1,
2521	4242	.suite = {
2522		- .aead = {
2523		- .enc = __VECS(hmac_sha224_des3_ede_cbc_enc_tv_temp)
2524		- }
	4243	+ .aead = __VECS(hmac_sha224_des3_ede_cbc_tv_temp)
2525	4244	}
2526	4245	}, {
2527	4246	.alg = "authenc(hmac(sha256),cbc(aes))",
2528	4247	.test = alg_test_aead,
2529	4248	.fips_allowed = 1,
2530	4249	.suite = {
2531		- .aead = {
2532		- .enc = __VECS(hmac_sha256_aes_cbc_enc_tv_temp)
2533		- }
	4250	+ .aead = __VECS(hmac_sha256_aes_cbc_tv_temp)
2534	4251	}
2535	4252	}, {
2536	4253	.alg = "authenc(hmac(sha256),cbc(des))",
2537	4254	.test = alg_test_aead,
2538	4255	.suite = {
2539		- .aead = {
2540		- .enc = __VECS(hmac_sha256_des_cbc_enc_tv_temp)
2541		- }
	4256	+ .aead = __VECS(hmac_sha256_des_cbc_tv_temp)
2542	4257	}
2543	4258	}, {
2544	4259	.alg = "authenc(hmac(sha256),cbc(des3_ede))",
2545	4260	.test = alg_test_aead,
2546	4261	.fips_allowed = 1,
2547	4262	.suite = {
2548		- .aead = {
2549		- .enc = __VECS(hmac_sha256_des3_ede_cbc_enc_tv_temp)
2550		- }
	4263	+ .aead = __VECS(hmac_sha256_des3_ede_cbc_tv_temp)
2551	4264	}
2552	4265	}, {
2553	4266	.alg = "authenc(hmac(sha256),ctr(aes))",
..	..	@@ -2561,18 +4274,14 @@
2561	4274	.alg = "authenc(hmac(sha384),cbc(des))",
2562	4275	.test = alg_test_aead,
2563	4276	.suite = {
2564		- .aead = {
2565		- .enc = __VECS(hmac_sha384_des_cbc_enc_tv_temp)
2566		- }
	4277	+ .aead = __VECS(hmac_sha384_des_cbc_tv_temp)
2567	4278	}
2568	4279	}, {
2569	4280	.alg = "authenc(hmac(sha384),cbc(des3_ede))",
2570	4281	.test = alg_test_aead,
2571	4282	.fips_allowed = 1,
2572	4283	.suite = {
2573		- .aead = {
2574		- .enc = __VECS(hmac_sha384_des3_ede_cbc_enc_tv_temp)
2575		- }
	4284	+ .aead = __VECS(hmac_sha384_des3_ede_cbc_tv_temp)
2576	4285	}
2577	4286	}, {
2578	4287	.alg = "authenc(hmac(sha384),ctr(aes))",
..	..	@@ -2587,26 +4296,20 @@
2587	4296	.fips_allowed = 1,
2588	4297	.test = alg_test_aead,
2589	4298	.suite = {
2590		- .aead = {
2591		- .enc = __VECS(hmac_sha512_aes_cbc_enc_tv_temp)
2592		- }
	4299	+ .aead = __VECS(hmac_sha512_aes_cbc_tv_temp)
2593	4300	}
2594	4301	}, {
2595	4302	.alg = "authenc(hmac(sha512),cbc(des))",
2596	4303	.test = alg_test_aead,
2597	4304	.suite = {
2598		- .aead = {
2599		- .enc = __VECS(hmac_sha512_des_cbc_enc_tv_temp)
2600		- }
	4305	+ .aead = __VECS(hmac_sha512_des_cbc_tv_temp)
2601	4306	}
2602	4307	}, {
2603	4308	.alg = "authenc(hmac(sha512),cbc(des3_ede))",
2604	4309	.test = alg_test_aead,
2605	4310	.fips_allowed = 1,
2606	4311	.suite = {
2607		- .aead = {
2608		- .enc = __VECS(hmac_sha512_des3_ede_cbc_enc_tv_temp)
2609		- }
	4312	+ .aead = __VECS(hmac_sha512_des3_ede_cbc_tv_temp)
2610	4313	}
2611	4314	}, {
2612	4315	.alg = "authenc(hmac(sha512),ctr(aes))",
..	..	@@ -2726,11 +4429,23 @@
2726	4429	.test = alg_test_null,
2727	4430	.fips_allowed = 1,
2728	4431	}, {
	4432	+ /* Same as cbc(sm4) except the key is stored in
	4433	+ * hardware secure memory which we reference by index
	4434	+ */
	4435	+ .alg = "cbc(psm4)",
	4436	+ .test = alg_test_null,
	4437	+ }, {
2729	4438	.alg = "cbc(serpent)",
2730	4439	.test = alg_test_skcipher,
2731	4440	.suite = {
2732	4441	.cipher = __VECS(serpent_cbc_tv_template)
2733	4442	},
	4443	+ }, {
	4444	+ .alg = "cbc(sm4)",
	4445	+ .test = alg_test_skcipher,
	4446	+ .suite = {
	4447	+ .cipher = __VECS(sm4_cbc_tv_template)
	4448	+ }
2734	4449	}, {
2735	4450	.alg = "cbc(twofish)",
2736	4451	.test = alg_test_skcipher,
..	..	@@ -2738,6 +4453,15 @@
2738	4453	.cipher = __VECS(tf_cbc_tv_template)
2739	4454	},
2740	4455	}, {
	4456	+#if IS_ENABLED(CONFIG_CRYPTO_PAES_S390)
	4457	+ .alg = "cbc-paes-s390",
	4458	+ .fips_allowed = 1,
	4459	+ .test = alg_test_skcipher,
	4460	+ .suite = {
	4461	+ .cipher = __VECS(aes_cbc_tv_template)
	4462	+ }
	4463	+ }, {
	4464	+#endif
2741	4465	.alg = "cbcmac(aes)",
2742	4466	.fips_allowed = 1,
2743	4467	.test = alg_test_hash,
..	..	@@ -2746,12 +4470,13 @@
2746	4470	}
2747	4471	}, {
2748	4472	.alg = "ccm(aes)",
	4473	+ .generic_driver = "ccm_base(ctr(aes-generic),cbcmac(aes-generic))",
2749	4474	.test = alg_test_aead,
2750	4475	.fips_allowed = 1,
2751	4476	.suite = {
2752	4477	.aead = {
2753		- .enc = __VECS(aes_ccm_enc_tv_template),
2754		- .dec = __VECS(aes_ccm_dec_tv_template)
	4478	+ ____VECS(aes_ccm_tv_template),
	4479	+ .einval_allowed = 1,
2755	4480	}
2756	4481	}
2757	4482	}, {
..	..	@@ -2761,6 +4486,12 @@
2761	4486	.suite = {
2762	4487	.cipher = __VECS(aes_cfb_tv_template)
2763	4488	},
	4489	+ }, {
	4490	+ .alg = "cfb(sm4)",
	4491	+ .test = alg_test_skcipher,
	4492	+ .suite = {
	4493	+ .cipher = __VECS(sm4_cfb_tv_template)
	4494	+ }
2764	4495	}, {
2765	4496	.alg = "chacha20",
2766	4497	.test = alg_test_skcipher,
..	..	@@ -2787,6 +4518,7 @@
2787	4518	}, {
2788	4519	.alg = "crc32",
2789	4520	.test = alg_test_hash,
	4521	+ .fips_allowed = 1,
2790	4522	.suite = {
2791	4523	.hash = __VECS(crc32_tv_template)
2792	4524	}
..	..	@@ -2856,10 +4588,23 @@
2856	4588	.test = alg_test_null,
2857	4589	.fips_allowed = 1,
2858	4590	}, {
	4591	+
	4592	+ /* Same as ctr(sm4) except the key is stored in
	4593	+ * hardware secure memory which we reference by index
	4594	+ */
	4595	+ .alg = "ctr(psm4)",
	4596	+ .test = alg_test_null,
	4597	+ }, {
2859	4598	.alg = "ctr(serpent)",
2860	4599	.test = alg_test_skcipher,
2861	4600	.suite = {
2862	4601	.cipher = __VECS(serpent_ctr_tv_template)
	4602	+ }
	4603	+ }, {
	4604	+ .alg = "ctr(sm4)",
	4605	+ .test = alg_test_skcipher,
	4606	+ .suite = {
	4607	+ .cipher = __VECS(sm4_ctr_tv_template)
2863	4608	}
2864	4609	}, {
2865	4610	.alg = "ctr(twofish)",
..	..	@@ -2868,11 +4613,28 @@
2868	4613	.cipher = __VECS(tf_ctr_tv_template)
2869	4614	}
2870	4615	}, {
	4616	+#if IS_ENABLED(CONFIG_CRYPTO_PAES_S390)
	4617	+ .alg = "ctr-paes-s390",
	4618	+ .fips_allowed = 1,
	4619	+ .test = alg_test_skcipher,
	4620	+ .suite = {
	4621	+ .cipher = __VECS(aes_ctr_tv_template)
	4622	+ }
	4623	+ }, {
	4624	+#endif
2871	4625	.alg = "cts(cbc(aes))",
2872	4626	.test = alg_test_skcipher,
	4627	+ .fips_allowed = 1,
2873	4628	.suite = {
2874	4629	.cipher = __VECS(cts_mode_tv_template)
2875	4630	}
	4631	+ }, {
	4632	+ /* Same as cts(cbc((aes)) except the key is stored in
	4633	+ * hardware secure memory which we reference by index
	4634	+ */
	4635	+ .alg = "cts(cbc(paes))",
	4636	+ .test = alg_test_null,
	4637	+ .fips_allowed = 1,
2876	4638	}, {
2877	4639	.alg = "curve25519",
2878	4640	.test = alg_test_kpp,
..	..	@@ -3035,6 +4797,7 @@
3035	4797	}
3036	4798	}, {
3037	4799	.alg = "ecb(arc4)",
	4800	+ .generic_driver = "ecb(arc4)-generic",
3038	4801	.test = alg_test_skcipher,
3039	4802	.suite = {
3040	4803	.cipher = __VECS(arc4_tv_template)
..	..	@@ -3151,6 +4914,15 @@
3151	4914	.cipher = __VECS(xtea_tv_template)
3152	4915	}
3153	4916	}, {
	4917	+#if IS_ENABLED(CONFIG_CRYPTO_PAES_S390)
	4918	+ .alg = "ecb-paes-s390",
	4919	+ .fips_allowed = 1,
	4920	+ .test = alg_test_skcipher,
	4921	+ .suite = {
	4922	+ .cipher = __VECS(aes_tv_template)
	4923	+ }
	4924	+ }, {
	4925	+#endif
3154	4926	.alg = "ecdh",
3155	4927	.test = alg_test_kpp,
3156	4928	.fips_allowed = 1,
..	..	@@ -3158,14 +4930,32 @@
3158	4930	.kpp = __VECS(ecdh_tv_template)
3159	4931	}
3160	4932	}, {
3161		- .alg = "gcm(aes)",
	4933	+ .alg = "ecrdsa",
	4934	+ .test = alg_test_akcipher,
	4935	+ .suite = {
	4936	+ .akcipher = __VECS(ecrdsa_tv_template)
	4937	+ }
	4938	+ }, {
	4939	+ .alg = "essiv(authenc(hmac(sha256),cbc(aes)),sha256)",
3162	4940	.test = alg_test_aead,
3163	4941	.fips_allowed = 1,
3164	4942	.suite = {
3165		- .aead = {
3166		- .enc = __VECS(aes_gcm_enc_tv_template),
3167		- .dec = __VECS(aes_gcm_dec_tv_template)
3168		- }
	4943	+ .aead = __VECS(essiv_hmac_sha256_aes_cbc_tv_temp)
	4944	+ }
	4945	+ }, {
	4946	+ .alg = "essiv(cbc(aes),sha256)",
	4947	+ .test = alg_test_skcipher,
	4948	+ .fips_allowed = 1,
	4949	+ .suite = {
	4950	+ .cipher = __VECS(essiv_aes_cbc_tv_template)
	4951	+ }
	4952	+ }, {
	4953	+ .alg = "gcm(aes)",
	4954	+ .generic_driver = "gcm_base(ctr(aes-generic),ghash-generic)",
	4955	+ .test = alg_test_aead,
	4956	+ .fips_allowed = 1,
	4957	+ .suite = {
	4958	+ .aead = __VECS(aes_gcm_tv_template)
3169	4959	}
3170	4960	}, {
3171	4961	.alg = "ghash",
..	..	@@ -3256,6 +5046,24 @@
3256	5046	.hash = __VECS(hmac_sha512_tv_template)
3257	5047	}
3258	5048	}, {
	5049	+ .alg = "hmac(sm3)",
	5050	+ .test = alg_test_hash,
	5051	+ .suite = {
	5052	+ .hash = __VECS(hmac_sm3_tv_template)
	5053	+ }
	5054	+ }, {
	5055	+ .alg = "hmac(streebog256)",
	5056	+ .test = alg_test_hash,
	5057	+ .suite = {
	5058	+ .hash = __VECS(hmac_streebog256_tv_template)
	5059	+ }
	5060	+ }, {
	5061	+ .alg = "hmac(streebog512)",
	5062	+ .test = alg_test_hash,
	5063	+ .suite = {
	5064	+ .hash = __VECS(hmac_streebog512_tv_template)
	5065	+ }
	5066	+ }, {
3259	5067	.alg = "jitterentropy_rng",
3260	5068	.fips_allowed = 1,
3261	5069	.test = alg_test_null,
..	..	@@ -3268,30 +5076,35 @@
3268	5076	}
3269	5077	}, {
3270	5078	.alg = "lrw(aes)",
	5079	+ .generic_driver = "lrw(ecb(aes-generic))",
3271	5080	.test = alg_test_skcipher,
3272	5081	.suite = {
3273	5082	.cipher = __VECS(aes_lrw_tv_template)
3274	5083	}
3275	5084	}, {
3276	5085	.alg = "lrw(camellia)",
	5086	+ .generic_driver = "lrw(ecb(camellia-generic))",
3277	5087	.test = alg_test_skcipher,
3278	5088	.suite = {
3279	5089	.cipher = __VECS(camellia_lrw_tv_template)
3280	5090	}
3281	5091	}, {
3282	5092	.alg = "lrw(cast6)",
	5093	+ .generic_driver = "lrw(ecb(cast6-generic))",
3283	5094	.test = alg_test_skcipher,
3284	5095	.suite = {
3285	5096	.cipher = __VECS(cast6_lrw_tv_template)
3286	5097	}
3287	5098	}, {
3288	5099	.alg = "lrw(serpent)",
	5100	+ .generic_driver = "lrw(ecb(serpent-generic))",
3289	5101	.test = alg_test_skcipher,
3290	5102	.suite = {
3291	5103	.cipher = __VECS(serpent_lrw_tv_template)
3292	5104	}
3293	5105	}, {
3294	5106	.alg = "lrw(twofish)",
	5107	+ .generic_driver = "lrw(ecb(twofish-generic))",
3295	5108	.test = alg_test_skcipher,
3296	5109	.suite = {
3297	5110	.cipher = __VECS(tf_lrw_tv_template)
..	..	@@ -3327,6 +5140,16 @@
3327	5140	}
3328	5141	}
3329	5142	}, {
	5143	+ .alg = "lzo-rle",
	5144	+ .test = alg_test_comp,
	5145	+ .fips_allowed = 1,
	5146	+ .suite = {
	5147	+ .comp = {
	5148	+ .comp = __VECS(lzorle_comp_tv_template),
	5149	+ .decomp = __VECS(lzorle_decomp_tv_template)
	5150	+ }
	5151	+ }
	5152	+ }, {
3330	5153	.alg = "md4",
3331	5154	.test = alg_test_hash,
3332	5155	.suite = {
..	..	@@ -3343,24 +5166,6 @@
3343	5166	.test = alg_test_hash,
3344	5167	.suite = {
3345	5168	.hash = __VECS(michael_mic_tv_template)
3346		- }
3347		- }, {
3348		- .alg = "morus1280",
3349		- .test = alg_test_aead,
3350		- .suite = {
3351		- .aead = {
3352		- .enc = __VECS(morus1280_enc_tv_template),
3353		- .dec = __VECS(morus1280_dec_tv_template),
3354		- }
3355		- }
3356		- }, {
3357		- .alg = "morus640",
3358		- .test = alg_test_aead,
3359		- .suite = {
3360		- .aead = {
3361		- .enc = __VECS(morus640_enc_tv_template),
3362		- .dec = __VECS(morus640_dec_tv_template),
3363		- }
3364	5169	}
3365	5170	}, {
3366	5171	.alg = "nhpoly1305",
..	..	@@ -3382,6 +5187,12 @@
3382	5187	.alg = "ofb(paes)",
3383	5188	.test = alg_test_null,
3384	5189	.fips_allowed = 1,
	5190	+ }, {
	5191	+ .alg = "ofb(sm4)",
	5192	+ .test = alg_test_skcipher,
	5193	+ .suite = {
	5194	+ .cipher = __VECS(sm4_ofb_tv_template)
	5195	+ }
3385	5196	}, {
3386	5197	.alg = "pcbc(fcrypt)",
3387	5198	.test = alg_test_skcipher,
..	..	@@ -3421,50 +5232,60 @@
3421	5232	.cipher = __VECS(aes_ctr_rfc3686_tv_template)
3422	5233	}
3423	5234	}, {
	5235	+ .alg = "rfc3686(ctr(sm4))",
	5236	+ .test = alg_test_skcipher,
	5237	+ .suite = {
	5238	+ .cipher = __VECS(sm4_ctr_rfc3686_tv_template)
	5239	+ }
	5240	+ }, {
3424	5241	.alg = "rfc4106(gcm(aes))",
	5242	+ .generic_driver = "rfc4106(gcm_base(ctr(aes-generic),ghash-generic))",
3425	5243	.test = alg_test_aead,
3426	5244	.fips_allowed = 1,
3427	5245	.suite = {
3428	5246	.aead = {
3429		- .enc = __VECS(aes_gcm_rfc4106_enc_tv_template),
3430		- .dec = __VECS(aes_gcm_rfc4106_dec_tv_template)
	5247	+ ____VECS(aes_gcm_rfc4106_tv_template),
	5248	+ .einval_allowed = 1,
	5249	+ .aad_iv = 1,
3431	5250	}
3432	5251	}
3433	5252	}, {
3434	5253	.alg = "rfc4309(ccm(aes))",
	5254	+ .generic_driver = "rfc4309(ccm_base(ctr(aes-generic),cbcmac(aes-generic)))",
3435	5255	.test = alg_test_aead,
3436	5256	.fips_allowed = 1,
3437	5257	.suite = {
3438	5258	.aead = {
3439		- .enc = __VECS(aes_ccm_rfc4309_enc_tv_template),
3440		- .dec = __VECS(aes_ccm_rfc4309_dec_tv_template)
	5259	+ ____VECS(aes_ccm_rfc4309_tv_template),
	5260	+ .einval_allowed = 1,
	5261	+ .aad_iv = 1,
3441	5262	}
3442	5263	}
3443	5264	}, {
3444	5265	.alg = "rfc4543(gcm(aes))",
	5266	+ .generic_driver = "rfc4543(gcm_base(ctr(aes-generic),ghash-generic))",
3445	5267	.test = alg_test_aead,
3446	5268	.suite = {
3447	5269	.aead = {
3448		- .enc = __VECS(aes_gcm_rfc4543_enc_tv_template),
3449		- .dec = __VECS(aes_gcm_rfc4543_dec_tv_template),
	5270	+ ____VECS(aes_gcm_rfc4543_tv_template),
	5271	+ .einval_allowed = 1,
	5272	+ .aad_iv = 1,
3450	5273	}
3451	5274	}
3452	5275	}, {
3453	5276	.alg = "rfc7539(chacha20,poly1305)",
3454	5277	.test = alg_test_aead,
3455	5278	.suite = {
3456		- .aead = {
3457		- .enc = __VECS(rfc7539_enc_tv_template),
3458		- .dec = __VECS(rfc7539_dec_tv_template),
3459		- }
	5279	+ .aead = __VECS(rfc7539_tv_template)
3460	5280	}
3461	5281	}, {
3462	5282	.alg = "rfc7539esp(chacha20,poly1305)",
3463	5283	.test = alg_test_aead,
3464	5284	.suite = {
3465	5285	.aead = {
3466		- .enc = __VECS(rfc7539esp_enc_tv_template),
3467		- .dec = __VECS(rfc7539esp_dec_tv_template),
	5286	+ ____VECS(rfc7539esp_tv_template),
	5287	+ .einval_allowed = 1,
	5288	+ .aad_iv = 1,
3468	5289	}
3469	5290	}
3470	5291	}, {
..	..	@@ -3568,10 +5389,28 @@
3568	5389	.hash = __VECS(sha512_tv_template)
3569	5390	}
3570	5391	}, {
	5392	+ .alg = "sm2",
	5393	+ .test = alg_test_akcipher,
	5394	+ .suite = {
	5395	+ .akcipher = __VECS(sm2_tv_template)
	5396	+ }
	5397	+ }, {
3571	5398	.alg = "sm3",
3572	5399	.test = alg_test_hash,
3573	5400	.suite = {
3574	5401	.hash = __VECS(sm3_tv_template)
	5402	+ }
	5403	+ }, {
	5404	+ .alg = "streebog256",
	5405	+ .test = alg_test_hash,
	5406	+ .suite = {
	5407	+ .hash = __VECS(streebog256_tv_template)
	5408	+ }
	5409	+ }, {
	5410	+ .alg = "streebog512",
	5411	+ .test = alg_test_hash,
	5412	+ .suite = {
	5413	+ .hash = __VECS(streebog512_tv_template)
3575	5414	}
3576	5415	}, {
3577	5416	.alg = "tgr128",
..	..	@@ -3635,6 +5474,7 @@
3635	5474	},
3636	5475	}, {
3637	5476	.alg = "xts(aes)",
	5477	+ .generic_driver = "xts(ecb(aes-generic))",
3638	5478	.test = alg_test_skcipher,
3639	5479	.fips_allowed = 1,
3640	5480	.suite = {
..	..	@@ -3642,12 +5482,14 @@
3642	5482	}
3643	5483	}, {
3644	5484	.alg = "xts(camellia)",
	5485	+ .generic_driver = "xts(ecb(camellia-generic))",
3645	5486	.test = alg_test_skcipher,
3646	5487	.suite = {
3647	5488	.cipher = __VECS(camellia_xts_tv_template)
3648	5489	}
3649	5490	}, {
3650	5491	.alg = "xts(cast6)",
	5492	+ .generic_driver = "xts(ecb(cast6-generic))",
3651	5493	.test = alg_test_skcipher,
3652	5494	.suite = {
3653	5495	.cipher = __VECS(cast6_xts_tv_template)
..	..	@@ -3661,17 +5503,28 @@
3661	5503	.fips_allowed = 1,
3662	5504	}, {
3663	5505	.alg = "xts(serpent)",
	5506	+ .generic_driver = "xts(ecb(serpent-generic))",
3664	5507	.test = alg_test_skcipher,
3665	5508	.suite = {
3666	5509	.cipher = __VECS(serpent_xts_tv_template)
3667	5510	}
3668	5511	}, {
3669	5512	.alg = "xts(twofish)",
	5513	+ .generic_driver = "xts(ecb(twofish-generic))",
3670	5514	.test = alg_test_skcipher,
3671	5515	.suite = {
3672	5516	.cipher = __VECS(tf_xts_tv_template)
3673	5517	}
3674	5518	}, {
	5519	+#if IS_ENABLED(CONFIG_CRYPTO_PAES_S390)
	5520	+ .alg = "xts-paes-s390",
	5521	+ .fips_allowed = 1,
	5522	+ .test = alg_test_skcipher,
	5523	+ .suite = {
	5524	+ .cipher = __VECS(aes_xts_tv_template)
	5525	+ }
	5526	+ }, {
	5527	+#endif
3675	5528	.alg = "xts4096(paes)",
3676	5529	.test = alg_test_null,
3677	5530	.fips_allowed = 1,
..	..	@@ -3679,6 +5532,13 @@
3679	5532	.alg = "xts512(paes)",
3680	5533	.test = alg_test_null,
3681	5534	.fips_allowed = 1,
	5535	+ }, {
	5536	+ .alg = "xxhash64",
	5537	+ .test = alg_test_hash,
	5538	+ .fips_allowed = 1,
	5539	+ .suite = {
	5540	+ .hash = __VECS(xxhash64_tv_template)
	5541	+ }
3682	5542	}, {
3683	5543	.alg = "zlib-deflate",
3684	5544	.test = alg_test_comp,
..	..	@@ -3702,17 +5562,9 @@
3702	5562	}
3703	5563	};
3704	5564
3705		-static bool alg_test_descs_checked;
3706		-
3707		-static void alg_test_descs_check_order(void)
	5565	+static void alg_check_test_descs_order(void)
3708	5566	{
3709	5567	int i;
3710		-
3711		- /* only check once */
3712		- if (alg_test_descs_checked)
3713		- return;
3714		-
3715		- alg_test_descs_checked = true;
3716	5568
3717	5569	for (i = 1; i < ARRAY_SIZE(alg_test_descs); i++) {
3718	5570	int diff = strcmp(alg_test_descs[i - 1].alg,
..	..	@@ -3729,6 +5581,29 @@
3729	5581	alg_test_descs[i].alg);
3730	5582	}
3731	5583	}
	5584	+}
	5585	+
	5586	+static void alg_check_testvec_configs(void)
	5587	+{
	5588	+ int i;
	5589	+
	5590	+ for (i = 0; i < ARRAY_SIZE(default_cipher_testvec_configs); i++)
	5591	+ WARN_ON(!valid_testvec_config(
	5592	+ &default_cipher_testvec_configs[i]));
	5593	+
	5594	+ for (i = 0; i < ARRAY_SIZE(default_hash_testvec_configs); i++)
	5595	+ WARN_ON(!valid_testvec_config(
	5596	+ &default_hash_testvec_configs[i]));
	5597	+}
	5598	+
	5599	+static void testmgr_onetime_init(void)
	5600	+{
	5601	+ alg_check_test_descs_order();
	5602	+ alg_check_testvec_configs();
	5603	+
	5604	+#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS
	5605	+ pr_warn("alg: extra crypto tests enabled. This is intended for developer use only.\n");
	5606	+#endif
3732	5607	}
3733	5608
3734	5609	static int alg_find_test(const char *alg)
..	..	@@ -3767,7 +5642,7 @@
3767	5642	return 0;
3768	5643	}
3769	5644
3770		- alg_test_descs_check_order();
	5645	+ DO_ONCE(testmgr_onetime_init);
3771	5646
3772	5647	if ((type & CRYPTO_ALG_TYPE_MASK) == CRYPTO_ALG_TYPE_CIPHER) {
3773	5648	char nalg[CRYPTO_MAX_ALG_NAME];
..	..	@@ -3805,8 +5680,11 @@
3805	5680	type, mask);
3806	5681
3807	5682	test_done:
3808		- if (fips_enabled && rc)
3809		- panic("%s: %s alg self test failed in fips mode!\n", driver, alg);
	5683	+ if (rc && (fips_enabled \|\| panic_on_fail)) {
	5684	+ fips_fail_notify();
	5685	+ panic("alg: self-tests for %s (%s) failed in %s mode!\n",
	5686	+ driver, alg, fips_enabled ? "fips" : "panic_on_fail");
	5687	+ }
3810	5688
3811	5689	if (fips_enabled && !rc)
3812	5690	pr_info("alg: self-tests for %s (%s) passed\n", driver, alg);