~hc/RK356X_SDK_RELEASE.git

..	..	@@ -1,36 +1,38 @@
	1	+// SPDX-License-Identifier: GPL-2.0-only
1	2	/*
2	3	* linux/arch/arm64/crypto/aes-glue.c - wrapper code for ARMv8 AES
3	4	*
4	5	* Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
5		- *
6		- * This program is free software; you can redistribute it and/or modify
7		- * it under the terms of the GNU General Public License version 2 as
8		- * published by the Free Software Foundation.
9	6	*/
10	7
11	8	#include <asm/neon.h>
12	9	#include <asm/hwcap.h>
13	10	#include <asm/simd.h>
14	11	#include <crypto/aes.h>
	12	+#include <crypto/ctr.h>
	13	+#include <crypto/sha.h>
15	14	#include <crypto/internal/hash.h>
16	15	#include <crypto/internal/simd.h>
17	16	#include <crypto/internal/skcipher.h>
	17	+#include <crypto/scatterwalk.h>
18	18	#include <linux/module.h>
19	19	#include <linux/cpufeature.h>
20	20	#include <crypto/xts.h>
21	21
22	22	#include "aes-ce-setkey.h"
23		-#include "aes-ctr-fallback.h"
24	23
25	24	#ifdef USE_V8_CRYPTO_EXTENSIONS
26	25	#define MODE "ce"
27	26	#define PRIO 300
28		-#define aes_setkey ce_aes_setkey
29	27	#define aes_expandkey ce_aes_expandkey
30	28	#define aes_ecb_encrypt ce_aes_ecb_encrypt
31	29	#define aes_ecb_decrypt ce_aes_ecb_decrypt
32	30	#define aes_cbc_encrypt ce_aes_cbc_encrypt
33	31	#define aes_cbc_decrypt ce_aes_cbc_decrypt
	32	+#define aes_cbc_cts_encrypt ce_aes_cbc_cts_encrypt
	33	+#define aes_cbc_cts_decrypt ce_aes_cbc_cts_decrypt
	34	+#define aes_essiv_cbc_encrypt ce_aes_essiv_cbc_encrypt
	35	+#define aes_essiv_cbc_decrypt ce_aes_essiv_cbc_decrypt
34	36	#define aes_ctr_encrypt ce_aes_ctr_encrypt
35	37	#define aes_xts_encrypt ce_aes_xts_encrypt
36	38	#define aes_xts_decrypt ce_aes_xts_decrypt
..	..	@@ -39,57 +41,81 @@
39	41	#else
40	42	#define MODE "neon"
41	43	#define PRIO 200
42		-#define aes_setkey crypto_aes_set_key
43		-#define aes_expandkey crypto_aes_expand_key
44	44	#define aes_ecb_encrypt neon_aes_ecb_encrypt
45	45	#define aes_ecb_decrypt neon_aes_ecb_decrypt
46	46	#define aes_cbc_encrypt neon_aes_cbc_encrypt
47	47	#define aes_cbc_decrypt neon_aes_cbc_decrypt
	48	+#define aes_cbc_cts_encrypt neon_aes_cbc_cts_encrypt
	49	+#define aes_cbc_cts_decrypt neon_aes_cbc_cts_decrypt
	50	+#define aes_essiv_cbc_encrypt neon_aes_essiv_cbc_encrypt
	51	+#define aes_essiv_cbc_decrypt neon_aes_essiv_cbc_decrypt
48	52	#define aes_ctr_encrypt neon_aes_ctr_encrypt
49	53	#define aes_xts_encrypt neon_aes_xts_encrypt
50	54	#define aes_xts_decrypt neon_aes_xts_decrypt
51	55	#define aes_mac_update neon_aes_mac_update
52	56	MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 NEON");
	57	+#endif
	58	+#if defined(USE_V8_CRYPTO_EXTENSIONS) \|\| !IS_ENABLED(CONFIG_CRYPTO_AES_ARM64_BS)
53	59	MODULE_ALIAS_CRYPTO("ecb(aes)");
54	60	MODULE_ALIAS_CRYPTO("cbc(aes)");
55	61	MODULE_ALIAS_CRYPTO("ctr(aes)");
56	62	MODULE_ALIAS_CRYPTO("xts(aes)");
	63	+#endif
	64	+MODULE_ALIAS_CRYPTO("cts(cbc(aes))");
	65	+MODULE_ALIAS_CRYPTO("essiv(cbc(aes),sha256)");
57	66	MODULE_ALIAS_CRYPTO("cmac(aes)");
58	67	MODULE_ALIAS_CRYPTO("xcbc(aes)");
59	68	MODULE_ALIAS_CRYPTO("cbcmac(aes)");
60		-#endif
61	69
62	70	MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
63	71	MODULE_LICENSE("GPL v2");
64	72
65	73	/* defined in aes-modes.S */
66		-asmlinkage void aes_ecb_encrypt(u8 out[], u8 const in[], u8 const rk[],
	74	+asmlinkage void aes_ecb_encrypt(u8 out[], u8 const in[], u32 const rk[],
67	75	int rounds, int blocks);
68		-asmlinkage void aes_ecb_decrypt(u8 out[], u8 const in[], u8 const rk[],
	76	+asmlinkage void aes_ecb_decrypt(u8 out[], u8 const in[], u32 const rk[],
69	77	int rounds, int blocks);
70	78
71		-asmlinkage void aes_cbc_encrypt(u8 out[], u8 const in[], u8 const rk[],
	79	+asmlinkage void aes_cbc_encrypt(u8 out[], u8 const in[], u32 const rk[],
72	80	int rounds, int blocks, u8 iv[]);
73		-asmlinkage void aes_cbc_decrypt(u8 out[], u8 const in[], u8 const rk[],
	81	+asmlinkage void aes_cbc_decrypt(u8 out[], u8 const in[], u32 const rk[],
74	82	int rounds, int blocks, u8 iv[]);
75	83
76		-asmlinkage void aes_ctr_encrypt(u8 out[], u8 const in[], u8 const rk[],
	84	+asmlinkage void aes_cbc_cts_encrypt(u8 out[], u8 const in[], u32 const rk[],
	85	+ int rounds, int bytes, u8 const iv[]);
	86	+asmlinkage void aes_cbc_cts_decrypt(u8 out[], u8 const in[], u32 const rk[],
	87	+ int rounds, int bytes, u8 const iv[]);
	88	+
	89	+asmlinkage void aes_ctr_encrypt(u8 out[], u8 const in[], u32 const rk[],
77	90	int rounds, int blocks, u8 ctr[]);
78	91
79		-asmlinkage void aes_xts_encrypt(u8 out[], u8 const in[], u8 const rk1[],
80		- int rounds, int blocks, u8 const rk2[], u8 iv[],
	92	+asmlinkage void aes_xts_encrypt(u8 out[], u8 const in[], u32 const rk1[],
	93	+ int rounds, int bytes, u32 const rk2[], u8 iv[],
81	94	int first);
82		-asmlinkage void aes_xts_decrypt(u8 out[], u8 const in[], u8 const rk1[],
83		- int rounds, int blocks, u8 const rk2[], u8 iv[],
	95	+asmlinkage void aes_xts_decrypt(u8 out[], u8 const in[], u32 const rk1[],
	96	+ int rounds, int bytes, u32 const rk2[], u8 iv[],
84	97	int first);
85	98
86		-asmlinkage void aes_mac_update(u8 const in[], u32 const rk[], int rounds,
87		- int blocks, u8 dg[], int enc_before,
88		- int enc_after);
	99	+asmlinkage void aes_essiv_cbc_encrypt(u8 out[], u8 const in[], u32 const rk1[],
	100	+ int rounds, int blocks, u8 iv[],
	101	+ u32 const rk2[]);
	102	+asmlinkage void aes_essiv_cbc_decrypt(u8 out[], u8 const in[], u32 const rk1[],
	103	+ int rounds, int blocks, u8 iv[],
	104	+ u32 const rk2[]);
	105	+
	106	+asmlinkage int aes_mac_update(u8 const in[], u32 const rk[], int rounds,
	107	+ int blocks, u8 dg[], int enc_before,
	108	+ int enc_after);
89	109
90	110	struct crypto_aes_xts_ctx {
91	111	struct crypto_aes_ctx key1;
92	112	struct crypto_aes_ctx __aligned(8) key2;
	113	+};
	114	+
	115	+struct crypto_aes_essiv_cbc_ctx {
	116	+ struct crypto_aes_ctx key1;
	117	+ struct crypto_aes_ctx __aligned(8) key2;
	118	+ struct crypto_shash *hash;
93	119	};
94	120
95	121	struct mac_tfm_ctx {
..	..	@@ -105,11 +131,13 @@
105	131	static int skcipher_aes_setkey(struct crypto_skcipher tfm, const u8 in_key,
106	132	unsigned int key_len)
107	133	{
108		- return aes_setkey(crypto_skcipher_tfm(tfm), in_key, key_len);
	134	+ struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
	135	+
	136	+ return aes_expandkey(ctx, in_key, key_len);
109	137	}
110	138
111		-static int xts_set_key(struct crypto_skcipher tfm, const u8 in_key,
112		- unsigned int key_len)
	139	+static int __maybe_unused xts_set_key(struct crypto_skcipher *tfm,
	140	+ const u8 *in_key, unsigned int key_len)
113	141	{
114	142	struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
115	143	int ret;
..	..	@@ -122,14 +150,27 @@
122	150	if (!ret)
123	151	ret = aes_expandkey(&ctx->key2, &in_key[key_len / 2],
124	152	key_len / 2);
125		- if (!ret)
126		- return 0;
127		-
128		- crypto_skcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
129		- return -EINVAL;
	153	+ return ret;
130	154	}
131	155
132		-static int ecb_encrypt(struct skcipher_request *req)
	156	+static int __maybe_unused essiv_cbc_set_key(struct crypto_skcipher *tfm,
	157	+ const u8 *in_key,
	158	+ unsigned int key_len)
	159	+{
	160	+ struct crypto_aes_essiv_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
	161	+ u8 digest[SHA256_DIGEST_SIZE];
	162	+ int ret;
	163	+
	164	+ ret = aes_expandkey(&ctx->key1, in_key, key_len);
	165	+ if (ret)
	166	+ return ret;
	167	+
	168	+ crypto_shash_tfm_digest(ctx->hash, in_key, key_len, digest);
	169	+
	170	+ return aes_expandkey(&ctx->key2, digest, sizeof(digest));
	171	+}
	172	+
	173	+static int __maybe_unused ecb_encrypt(struct skcipher_request *req)
133	174	{
134	175	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
135	176	struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
..	..	@@ -142,14 +183,14 @@
142	183	while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
143	184	kernel_neon_begin();
144	185	aes_ecb_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
145		- (u8 *)ctx->key_enc, rounds, blocks);
	186	+ ctx->key_enc, rounds, blocks);
146	187	kernel_neon_end();
147	188	err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
148	189	}
149	190	return err;
150	191	}
151	192
152		-static int ecb_decrypt(struct skcipher_request *req)
	193	+static int __maybe_unused ecb_decrypt(struct skcipher_request *req)
153	194	{
154	195	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
155	196	struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
..	..	@@ -162,54 +203,246 @@
162	203	while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
163	204	kernel_neon_begin();
164	205	aes_ecb_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
165		- (u8 *)ctx->key_dec, rounds, blocks);
	206	+ ctx->key_dec, rounds, blocks);
166	207	kernel_neon_end();
167	208	err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
168	209	}
169	210	return err;
170	211	}
171	212
172		-static int cbc_encrypt(struct skcipher_request *req)
	213	+static int cbc_encrypt_walk(struct skcipher_request *req,
	214	+ struct skcipher_walk *walk)
	215	+{
	216	+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
	217	+ struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
	218	+ int err = 0, rounds = 6 + ctx->key_length / 4;
	219	+ unsigned int blocks;
	220	+
	221	+ while ((blocks = (walk->nbytes / AES_BLOCK_SIZE))) {
	222	+ kernel_neon_begin();
	223	+ aes_cbc_encrypt(walk->dst.virt.addr, walk->src.virt.addr,
	224	+ ctx->key_enc, rounds, blocks, walk->iv);
	225	+ kernel_neon_end();
	226	+ err = skcipher_walk_done(walk, walk->nbytes % AES_BLOCK_SIZE);
	227	+ }
	228	+ return err;
	229	+}
	230	+
	231	+static int __maybe_unused cbc_encrypt(struct skcipher_request *req)
	232	+{
	233	+ struct skcipher_walk walk;
	234	+ int err;
	235	+
	236	+ err = skcipher_walk_virt(&walk, req, false);
	237	+ if (err)
	238	+ return err;
	239	+ return cbc_encrypt_walk(req, &walk);
	240	+}
	241	+
	242	+static int cbc_decrypt_walk(struct skcipher_request *req,
	243	+ struct skcipher_walk *walk)
	244	+{
	245	+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
	246	+ struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
	247	+ int err = 0, rounds = 6 + ctx->key_length / 4;
	248	+ unsigned int blocks;
	249	+
	250	+ while ((blocks = (walk->nbytes / AES_BLOCK_SIZE))) {
	251	+ kernel_neon_begin();
	252	+ aes_cbc_decrypt(walk->dst.virt.addr, walk->src.virt.addr,
	253	+ ctx->key_dec, rounds, blocks, walk->iv);
	254	+ kernel_neon_end();
	255	+ err = skcipher_walk_done(walk, walk->nbytes % AES_BLOCK_SIZE);
	256	+ }
	257	+ return err;
	258	+}
	259	+
	260	+static int __maybe_unused cbc_decrypt(struct skcipher_request *req)
	261	+{
	262	+ struct skcipher_walk walk;
	263	+ int err;
	264	+
	265	+ err = skcipher_walk_virt(&walk, req, false);
	266	+ if (err)
	267	+ return err;
	268	+ return cbc_decrypt_walk(req, &walk);
	269	+}
	270	+
	271	+static int cts_cbc_encrypt(struct skcipher_request *req)
173	272	{
174	273	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
175	274	struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
176	275	int err, rounds = 6 + ctx->key_length / 4;
	276	+ int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2;
	277	+ struct scatterlist src = req->src, dst = req->dst;
	278	+ struct scatterlist sg_src[2], sg_dst[2];
	279	+ struct skcipher_request subreq;
	280	+ struct skcipher_walk walk;
	281	+
	282	+ skcipher_request_set_tfm(&subreq, tfm);
	283	+ skcipher_request_set_callback(&subreq, skcipher_request_flags(req),
	284	+ NULL, NULL);
	285	+
	286	+ if (req->cryptlen <= AES_BLOCK_SIZE) {
	287	+ if (req->cryptlen < AES_BLOCK_SIZE)
	288	+ return -EINVAL;
	289	+ cbc_blocks = 1;
	290	+ }
	291	+
	292	+ if (cbc_blocks > 0) {
	293	+ skcipher_request_set_crypt(&subreq, req->src, req->dst,
	294	+ cbc_blocks * AES_BLOCK_SIZE,
	295	+ req->iv);
	296	+
	297	+ err = skcipher_walk_virt(&walk, &subreq, false) ?:
	298	+ cbc_encrypt_walk(&subreq, &walk);
	299	+ if (err)
	300	+ return err;
	301	+
	302	+ if (req->cryptlen == AES_BLOCK_SIZE)
	303	+ return 0;
	304	+
	305	+ dst = src = scatterwalk_ffwd(sg_src, req->src, subreq.cryptlen);
	306	+ if (req->dst != req->src)
	307	+ dst = scatterwalk_ffwd(sg_dst, req->dst,
	308	+ subreq.cryptlen);
	309	+ }
	310	+
	311	+ /* handle ciphertext stealing */
	312	+ skcipher_request_set_crypt(&subreq, src, dst,
	313	+ req->cryptlen - cbc_blocks * AES_BLOCK_SIZE,
	314	+ req->iv);
	315	+
	316	+ err = skcipher_walk_virt(&walk, &subreq, false);
	317	+ if (err)
	318	+ return err;
	319	+
	320	+ kernel_neon_begin();
	321	+ aes_cbc_cts_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
	322	+ ctx->key_enc, rounds, walk.nbytes, walk.iv);
	323	+ kernel_neon_end();
	324	+
	325	+ return skcipher_walk_done(&walk, 0);
	326	+}
	327	+
	328	+static int cts_cbc_decrypt(struct skcipher_request *req)
	329	+{
	330	+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
	331	+ struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
	332	+ int err, rounds = 6 + ctx->key_length / 4;
	333	+ int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2;
	334	+ struct scatterlist src = req->src, dst = req->dst;
	335	+ struct scatterlist sg_src[2], sg_dst[2];
	336	+ struct skcipher_request subreq;
	337	+ struct skcipher_walk walk;
	338	+
	339	+ skcipher_request_set_tfm(&subreq, tfm);
	340	+ skcipher_request_set_callback(&subreq, skcipher_request_flags(req),
	341	+ NULL, NULL);
	342	+
	343	+ if (req->cryptlen <= AES_BLOCK_SIZE) {
	344	+ if (req->cryptlen < AES_BLOCK_SIZE)
	345	+ return -EINVAL;
	346	+ cbc_blocks = 1;
	347	+ }
	348	+
	349	+ if (cbc_blocks > 0) {
	350	+ skcipher_request_set_crypt(&subreq, req->src, req->dst,
	351	+ cbc_blocks * AES_BLOCK_SIZE,
	352	+ req->iv);
	353	+
	354	+ err = skcipher_walk_virt(&walk, &subreq, false) ?:
	355	+ cbc_decrypt_walk(&subreq, &walk);
	356	+ if (err)
	357	+ return err;
	358	+
	359	+ if (req->cryptlen == AES_BLOCK_SIZE)
	360	+ return 0;
	361	+
	362	+ dst = src = scatterwalk_ffwd(sg_src, req->src, subreq.cryptlen);
	363	+ if (req->dst != req->src)
	364	+ dst = scatterwalk_ffwd(sg_dst, req->dst,
	365	+ subreq.cryptlen);
	366	+ }
	367	+
	368	+ /* handle ciphertext stealing */
	369	+ skcipher_request_set_crypt(&subreq, src, dst,
	370	+ req->cryptlen - cbc_blocks * AES_BLOCK_SIZE,
	371	+ req->iv);
	372	+
	373	+ err = skcipher_walk_virt(&walk, &subreq, false);
	374	+ if (err)
	375	+ return err;
	376	+
	377	+ kernel_neon_begin();
	378	+ aes_cbc_cts_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
	379	+ ctx->key_dec, rounds, walk.nbytes, walk.iv);
	380	+ kernel_neon_end();
	381	+
	382	+ return skcipher_walk_done(&walk, 0);
	383	+}
	384	+
	385	+static int __maybe_unused essiv_cbc_init_tfm(struct crypto_skcipher *tfm)
	386	+{
	387	+ struct crypto_aes_essiv_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
	388	+
	389	+ ctx->hash = crypto_alloc_shash("sha256", 0, 0);
	390	+
	391	+ return PTR_ERR_OR_ZERO(ctx->hash);
	392	+}
	393	+
	394	+static void __maybe_unused essiv_cbc_exit_tfm(struct crypto_skcipher *tfm)
	395	+{
	396	+ struct crypto_aes_essiv_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
	397	+
	398	+ crypto_free_shash(ctx->hash);
	399	+}
	400	+
	401	+static int __maybe_unused essiv_cbc_encrypt(struct skcipher_request *req)
	402	+{
	403	+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
	404	+ struct crypto_aes_essiv_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
	405	+ int err, rounds = 6 + ctx->key1.key_length / 4;
177	406	struct skcipher_walk walk;
178	407	unsigned int blocks;
179	408
180	409	err = skcipher_walk_virt(&walk, req, false);
181	410
182		- while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
	411	+ blocks = walk.nbytes / AES_BLOCK_SIZE;
	412	+ if (blocks) {
183	413	kernel_neon_begin();
184		- aes_cbc_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
185		- (u8 *)ctx->key_enc, rounds, blocks, walk.iv);
	414	+ aes_essiv_cbc_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
	415	+ ctx->key1.key_enc, rounds, blocks,
	416	+ req->iv, ctx->key2.key_enc);
186	417	kernel_neon_end();
187	418	err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
188	419	}
189		- return err;
	420	+ return err ?: cbc_encrypt_walk(req, &walk);
190	421	}
191	422
192		-static int cbc_decrypt(struct skcipher_request *req)
	423	+static int __maybe_unused essiv_cbc_decrypt(struct skcipher_request *req)
193	424	{
194	425	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
195		- struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
196		- int err, rounds = 6 + ctx->key_length / 4;
	426	+ struct crypto_aes_essiv_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
	427	+ int err, rounds = 6 + ctx->key1.key_length / 4;
197	428	struct skcipher_walk walk;
198	429	unsigned int blocks;
199	430
200	431	err = skcipher_walk_virt(&walk, req, false);
201	432
202		- while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
	433	+ blocks = walk.nbytes / AES_BLOCK_SIZE;
	434	+ if (blocks) {
203	435	kernel_neon_begin();
204		- aes_cbc_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
205		- (u8 *)ctx->key_dec, rounds, blocks, walk.iv);
	436	+ aes_essiv_cbc_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
	437	+ ctx->key1.key_dec, rounds, blocks,
	438	+ req->iv, ctx->key2.key_enc);
206	439	kernel_neon_end();
207	440	err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
208	441	}
209		- return err;
	442	+ return err ?: cbc_decrypt_walk(req, &walk);
210	443	}
211	444
212		-static int ctr_encrypt(struct skcipher_request *req)
	445	+static int __maybe_unused ctr_encrypt(struct skcipher_request *req)
213	446	{
214	447	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
215	448	struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
..	..	@@ -222,7 +455,7 @@
222	455	while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
223	456	kernel_neon_begin();
224	457	aes_ctr_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
225		- (u8 *)ctx->key_enc, rounds, blocks, walk.iv);
	458	+ ctx->key_enc, rounds, blocks, walk.iv);
226	459	kernel_neon_end();
227	460	err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
228	461	}
..	..	@@ -238,7 +471,7 @@
238	471	blocks = -1;
239	472
240	473	kernel_neon_begin();
241		- aes_ctr_encrypt(tail, NULL, (u8 *)ctx->key_enc, rounds,
	474	+ aes_ctr_encrypt(tail, NULL, ctx->key_enc, rounds,
242	475	blocks, walk.iv);
243	476	kernel_neon_end();
244	477	crypto_xor_cpy(tdst, tsrc, tail, nbytes);
..	..	@@ -248,67 +481,157 @@
248	481	return err;
249	482	}
250	483
251		-static int ctr_encrypt_sync(struct skcipher_request *req)
252		-{
253		- struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
254		- struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
255		-
256		- if (!may_use_simd())
257		- return aes_ctr_encrypt_fallback(ctx, req);
258		-
259		- return ctr_encrypt(req);
260		-}
261		-
262		-static int xts_encrypt(struct skcipher_request *req)
	484	+static int __maybe_unused xts_encrypt(struct skcipher_request *req)
263	485	{
264	486	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
265	487	struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
266	488	int err, first, rounds = 6 + ctx->key1.key_length / 4;
	489	+ int tail = req->cryptlen % AES_BLOCK_SIZE;
	490	+ struct scatterlist sg_src[2], sg_dst[2];
	491	+ struct skcipher_request subreq;
	492	+ struct scatterlist src, dst;
267	493	struct skcipher_walk walk;
268		- unsigned int blocks;
	494	+
	495	+ if (req->cryptlen < AES_BLOCK_SIZE)
	496	+ return -EINVAL;
269	497
270	498	err = skcipher_walk_virt(&walk, req, false);
271	499
272		- for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) {
	500	+ if (unlikely(tail > 0 && walk.nbytes < walk.total)) {
	501	+ int xts_blocks = DIV_ROUND_UP(req->cryptlen,
	502	+ AES_BLOCK_SIZE) - 2;
	503	+
	504	+ skcipher_walk_abort(&walk);
	505	+
	506	+ skcipher_request_set_tfm(&subreq, tfm);
	507	+ skcipher_request_set_callback(&subreq,
	508	+ skcipher_request_flags(req),
	509	+ NULL, NULL);
	510	+ skcipher_request_set_crypt(&subreq, req->src, req->dst,
	511	+ xts_blocks * AES_BLOCK_SIZE,
	512	+ req->iv);
	513	+ req = &subreq;
	514	+ err = skcipher_walk_virt(&walk, req, false);
	515	+ } else {
	516	+ tail = 0;
	517	+ }
	518	+
	519	+ for (first = 1; walk.nbytes >= AES_BLOCK_SIZE; first = 0) {
	520	+ int nbytes = walk.nbytes;
	521	+
	522	+ if (walk.nbytes < walk.total)
	523	+ nbytes &= ~(AES_BLOCK_SIZE - 1);
	524	+
273	525	kernel_neon_begin();
274	526	aes_xts_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
275		- (u8 *)ctx->key1.key_enc, rounds, blocks,
276		- (u8 *)ctx->key2.key_enc, walk.iv, first);
	527	+ ctx->key1.key_enc, rounds, nbytes,
	528	+ ctx->key2.key_enc, walk.iv, first);
277	529	kernel_neon_end();
278		- err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
	530	+ err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
279	531	}
280	532
281		- return err;
	533	+ if (err \|\| likely(!tail))
	534	+ return err;
	535	+
	536	+ dst = src = scatterwalk_ffwd(sg_src, req->src, req->cryptlen);
	537	+ if (req->dst != req->src)
	538	+ dst = scatterwalk_ffwd(sg_dst, req->dst, req->cryptlen);
	539	+
	540	+ skcipher_request_set_crypt(req, src, dst, AES_BLOCK_SIZE + tail,
	541	+ req->iv);
	542	+
	543	+ err = skcipher_walk_virt(&walk, &subreq, false);
	544	+ if (err)
	545	+ return err;
	546	+
	547	+ kernel_neon_begin();
	548	+ aes_xts_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
	549	+ ctx->key1.key_enc, rounds, walk.nbytes,
	550	+ ctx->key2.key_enc, walk.iv, first);
	551	+ kernel_neon_end();
	552	+
	553	+ return skcipher_walk_done(&walk, 0);
282	554	}
283	555
284		-static int xts_decrypt(struct skcipher_request *req)
	556	+static int __maybe_unused xts_decrypt(struct skcipher_request *req)
285	557	{
286	558	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
287	559	struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
288	560	int err, first, rounds = 6 + ctx->key1.key_length / 4;
	561	+ int tail = req->cryptlen % AES_BLOCK_SIZE;
	562	+ struct scatterlist sg_src[2], sg_dst[2];
	563	+ struct skcipher_request subreq;
	564	+ struct scatterlist src, dst;
289	565	struct skcipher_walk walk;
290		- unsigned int blocks;
	566	+
	567	+ if (req->cryptlen < AES_BLOCK_SIZE)
	568	+ return -EINVAL;
291	569
292	570	err = skcipher_walk_virt(&walk, req, false);
293	571
294		- for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) {
295		- kernel_neon_begin();
296		- aes_xts_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
297		- (u8 *)ctx->key1.key_dec, rounds, blocks,
298		- (u8 *)ctx->key2.key_enc, walk.iv, first);
299		- kernel_neon_end();
300		- err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
	572	+ if (unlikely(tail > 0 && walk.nbytes < walk.total)) {
	573	+ int xts_blocks = DIV_ROUND_UP(req->cryptlen,
	574	+ AES_BLOCK_SIZE) - 2;
	575	+
	576	+ skcipher_walk_abort(&walk);
	577	+
	578	+ skcipher_request_set_tfm(&subreq, tfm);
	579	+ skcipher_request_set_callback(&subreq,
	580	+ skcipher_request_flags(req),
	581	+ NULL, NULL);
	582	+ skcipher_request_set_crypt(&subreq, req->src, req->dst,
	583	+ xts_blocks * AES_BLOCK_SIZE,
	584	+ req->iv);
	585	+ req = &subreq;
	586	+ err = skcipher_walk_virt(&walk, req, false);
	587	+ } else {
	588	+ tail = 0;
301	589	}
302	590
303		- return err;
	591	+ for (first = 1; walk.nbytes >= AES_BLOCK_SIZE; first = 0) {
	592	+ int nbytes = walk.nbytes;
	593	+
	594	+ if (walk.nbytes < walk.total)
	595	+ nbytes &= ~(AES_BLOCK_SIZE - 1);
	596	+
	597	+ kernel_neon_begin();
	598	+ aes_xts_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
	599	+ ctx->key1.key_dec, rounds, nbytes,
	600	+ ctx->key2.key_enc, walk.iv, first);
	601	+ kernel_neon_end();
	602	+ err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
	603	+ }
	604	+
	605	+ if (err \|\| likely(!tail))
	606	+ return err;
	607	+
	608	+ dst = src = scatterwalk_ffwd(sg_src, req->src, req->cryptlen);
	609	+ if (req->dst != req->src)
	610	+ dst = scatterwalk_ffwd(sg_dst, req->dst, req->cryptlen);
	611	+
	612	+ skcipher_request_set_crypt(req, src, dst, AES_BLOCK_SIZE + tail,
	613	+ req->iv);
	614	+
	615	+ err = skcipher_walk_virt(&walk, &subreq, false);
	616	+ if (err)
	617	+ return err;
	618	+
	619	+
	620	+ kernel_neon_begin();
	621	+ aes_xts_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
	622	+ ctx->key1.key_dec, rounds, walk.nbytes,
	623	+ ctx->key2.key_enc, walk.iv, first);
	624	+ kernel_neon_end();
	625	+
	626	+ return skcipher_walk_done(&walk, 0);
304	627	}
305	628
306	629	static struct skcipher_alg aes_algs[] = { {
	630	+#if defined(USE_V8_CRYPTO_EXTENSIONS) \|\| !IS_ENABLED(CONFIG_CRYPTO_AES_ARM64_BS)
307	631	.base = {
308		- .cra_name = "__ecb(aes)",
309		- .cra_driver_name = "__ecb-aes-" MODE,
	632	+ .cra_name = "ecb(aes)",
	633	+ .cra_driver_name = "ecb-aes-" MODE,
310	634	.cra_priority = PRIO,
311		- .cra_flags = CRYPTO_ALG_INTERNAL,
312	635	.cra_blocksize = AES_BLOCK_SIZE,
313	636	.cra_ctxsize = sizeof(struct crypto_aes_ctx),
314	637	.cra_module = THIS_MODULE,
..	..	@@ -320,10 +643,9 @@
320	643	.decrypt = ecb_decrypt,
321	644	}, {
322	645	.base = {
323		- .cra_name = "__cbc(aes)",
324		- .cra_driver_name = "__cbc-aes-" MODE,
	646	+ .cra_name = "cbc(aes)",
	647	+ .cra_driver_name = "cbc-aes-" MODE,
325	648	.cra_priority = PRIO,
326		- .cra_flags = CRYPTO_ALG_INTERNAL,
327	649	.cra_blocksize = AES_BLOCK_SIZE,
328	650	.cra_ctxsize = sizeof(struct crypto_aes_ctx),
329	651	.cra_module = THIS_MODULE,
..	..	@@ -336,10 +658,9 @@
336	658	.decrypt = cbc_decrypt,
337	659	}, {
338	660	.base = {
339		- .cra_name = "__ctr(aes)",
340		- .cra_driver_name = "__ctr-aes-" MODE,
	661	+ .cra_name = "ctr(aes)",
	662	+ .cra_driver_name = "ctr-aes-" MODE,
341	663	.cra_priority = PRIO,
342		- .cra_flags = CRYPTO_ALG_INTERNAL,
343	664	.cra_blocksize = 1,
344	665	.cra_ctxsize = sizeof(struct crypto_aes_ctx),
345	666	.cra_module = THIS_MODULE,
..	..	@@ -353,26 +674,9 @@
353	674	.decrypt = ctr_encrypt,
354	675	}, {
355	676	.base = {
356		- .cra_name = "ctr(aes)",
357		- .cra_driver_name = "ctr-aes-" MODE,
358		- .cra_priority = PRIO - 1,
359		- .cra_blocksize = 1,
360		- .cra_ctxsize = sizeof(struct crypto_aes_ctx),
361		- .cra_module = THIS_MODULE,
362		- },
363		- .min_keysize = AES_MIN_KEY_SIZE,
364		- .max_keysize = AES_MAX_KEY_SIZE,
365		- .ivsize = AES_BLOCK_SIZE,
366		- .chunksize = AES_BLOCK_SIZE,
367		- .setkey = skcipher_aes_setkey,
368		- .encrypt = ctr_encrypt_sync,
369		- .decrypt = ctr_encrypt_sync,
370		-}, {
371		- .base = {
372		- .cra_name = "__xts(aes)",
373		- .cra_driver_name = "__xts-aes-" MODE,
	677	+ .cra_name = "xts(aes)",
	678	+ .cra_driver_name = "xts-aes-" MODE,
374	679	.cra_priority = PRIO,
375		- .cra_flags = CRYPTO_ALG_INTERNAL,
376	680	.cra_blocksize = AES_BLOCK_SIZE,
377	681	.cra_ctxsize = sizeof(struct crypto_aes_xts_ctx),
378	682	.cra_module = THIS_MODULE,
..	..	@@ -380,22 +684,52 @@
380	684	.min_keysize = 2 * AES_MIN_KEY_SIZE,
381	685	.max_keysize = 2 * AES_MAX_KEY_SIZE,
382	686	.ivsize = AES_BLOCK_SIZE,
	687	+ .walksize = 2 * AES_BLOCK_SIZE,
383	688	.setkey = xts_set_key,
384	689	.encrypt = xts_encrypt,
385	690	.decrypt = xts_decrypt,
	691	+}, {
	692	+#endif
	693	+ .base = {
	694	+ .cra_name = "cts(cbc(aes))",
	695	+ .cra_driver_name = "cts-cbc-aes-" MODE,
	696	+ .cra_priority = PRIO,
	697	+ .cra_blocksize = AES_BLOCK_SIZE,
	698	+ .cra_ctxsize = sizeof(struct crypto_aes_ctx),
	699	+ .cra_module = THIS_MODULE,
	700	+ },
	701	+ .min_keysize = AES_MIN_KEY_SIZE,
	702	+ .max_keysize = AES_MAX_KEY_SIZE,
	703	+ .ivsize = AES_BLOCK_SIZE,
	704	+ .walksize = 2 * AES_BLOCK_SIZE,
	705	+ .setkey = skcipher_aes_setkey,
	706	+ .encrypt = cts_cbc_encrypt,
	707	+ .decrypt = cts_cbc_decrypt,
	708	+}, {
	709	+ .base = {
	710	+ .cra_name = "essiv(cbc(aes),sha256)",
	711	+ .cra_driver_name = "essiv-cbc-aes-sha256-" MODE,
	712	+ .cra_priority = PRIO + 1,
	713	+ .cra_blocksize = AES_BLOCK_SIZE,
	714	+ .cra_ctxsize = sizeof(struct crypto_aes_essiv_cbc_ctx),
	715	+ .cra_module = THIS_MODULE,
	716	+ },
	717	+ .min_keysize = AES_MIN_KEY_SIZE,
	718	+ .max_keysize = AES_MAX_KEY_SIZE,
	719	+ .ivsize = AES_BLOCK_SIZE,
	720	+ .setkey = essiv_cbc_set_key,
	721	+ .encrypt = essiv_cbc_encrypt,
	722	+ .decrypt = essiv_cbc_decrypt,
	723	+ .init = essiv_cbc_init_tfm,
	724	+ .exit = essiv_cbc_exit_tfm,
386	725	} };
387	726
388	727	static int cbcmac_setkey(struct crypto_shash tfm, const u8 in_key,
389	728	unsigned int key_len)
390	729	{
391	730	struct mac_tfm_ctx *ctx = crypto_shash_ctx(tfm);
392		- int err;
393	731
394		- err = aes_expandkey(&ctx->key, in_key, key_len);
395		- if (err)
396		- crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
397		-
398		- return err;
	732	+ return aes_expandkey(&ctx->key, in_key, key_len);
399	733	}
400	734
401	735	static void cmac_gf128_mul_by_x(be128 y, const be128 x)
..	..	@@ -412,7 +746,6 @@
412	746	{
413	747	struct mac_tfm_ctx *ctx = crypto_shash_ctx(tfm);
414	748	be128 consts = (be128 )ctx->consts;
415		- u8 rk = (u8 )ctx->key.key_enc;
416	749	int rounds = 6 + key_len / 4;
417	750	int err;
418	751
..	..	@@ -422,7 +755,8 @@
422	755
423	756	/* encrypt the zero vector */
424	757	kernel_neon_begin();
425		- aes_ecb_encrypt(ctx->consts, (u8[AES_BLOCK_SIZE]){}, rk, rounds, 1);
	758	+ aes_ecb_encrypt(ctx->consts, (u8[AES_BLOCK_SIZE]){}, ctx->key.key_enc,
	759	+ rounds, 1);
426	760	kernel_neon_end();
427	761
428	762	cmac_gf128_mul_by_x(consts, consts);
..	..	@@ -441,7 +775,6 @@
441	775	};
442	776
443	777	struct mac_tfm_ctx *ctx = crypto_shash_ctx(tfm);
444		- u8 rk = (u8 )ctx->key.key_enc;
445	778	int rounds = 6 + key_len / 4;
446	779	u8 key[AES_BLOCK_SIZE];
447	780	int err;
..	..	@@ -451,8 +784,8 @@
451	784	return err;
452	785
453	786	kernel_neon_begin();
454		- aes_ecb_encrypt(key, ks[0], rk, rounds, 1);
455		- aes_ecb_encrypt(ctx->consts, ks[1], rk, rounds, 2);
	787	+ aes_ecb_encrypt(key, ks[0], ctx->key.key_enc, rounds, 1);
	788	+ aes_ecb_encrypt(ctx->consts, ks[1], ctx->key.key_enc, rounds, 2);
456	789	kernel_neon_end();
457	790
458	791	return cbcmac_setkey(tfm, key, sizeof(key));
..	..	@@ -473,22 +806,28 @@
473	806	{
474	807	int rounds = 6 + ctx->key_length / 4;
475	808
476		- if (may_use_simd()) {
477		- kernel_neon_begin();
478		- aes_mac_update(in, ctx->key_enc, rounds, blocks, dg, enc_before,
479		- enc_after);
480		- kernel_neon_end();
	809	+ if (crypto_simd_usable()) {
	810	+ int rem;
	811	+
	812	+ do {
	813	+ kernel_neon_begin();
	814	+ rem = aes_mac_update(in, ctx->key_enc, rounds, blocks,
	815	+ dg, enc_before, enc_after);
	816	+ kernel_neon_end();
	817	+ in += (blocks - rem) * AES_BLOCK_SIZE;
	818	+ blocks = rem;
	819	+ enc_before = 0;
	820	+ } while (blocks);
481	821	} else {
482	822	if (enc_before)
483		- __aes_arm64_encrypt(ctx->key_enc, dg, dg, rounds);
	823	+ aes_encrypt(ctx, dg, dg);
484	824
485	825	while (blocks--) {
486	826	crypto_xor(dg, in, AES_BLOCK_SIZE);
487	827	in += AES_BLOCK_SIZE;
488	828
489	829	if (blocks \|\| enc_after)
490		- __aes_arm64_encrypt(ctx->key_enc, dg, dg,
491		- rounds);
	830	+ aes_encrypt(ctx, dg, dg);
492	831	}
493	832	}
494	833	}
..	..	@@ -538,7 +877,7 @@
538	877	struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
539	878	struct mac_desc_ctx *ctx = shash_desc_ctx(desc);
540	879
541		- mac_do_update(&tctx->key, NULL, 0, ctx->dg, 1, 0);
	880	+ mac_do_update(&tctx->key, NULL, 0, ctx->dg, (ctx->len != 0), 0);
542	881
543	882	memcpy(out, ctx->dg, AES_BLOCK_SIZE);
544	883
..	..	@@ -609,28 +948,15 @@
609	948	.descsize = sizeof(struct mac_desc_ctx),
610	949	} };
611	950
612		-static struct simd_skcipher_alg *aes_simd_algs[ARRAY_SIZE(aes_algs)];
613		-
614	951	static void aes_exit(void)
615	952	{
616		- int i;
617		-
618		- for (i = 0; i < ARRAY_SIZE(aes_simd_algs); i++)
619		- if (aes_simd_algs[i])
620		- simd_skcipher_free(aes_simd_algs[i]);
621		-
622	953	crypto_unregister_shashes(mac_algs, ARRAY_SIZE(mac_algs));
623	954	crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
624	955	}
625	956
626	957	static int __init aes_init(void)
627	958	{
628		- struct simd_skcipher_alg *simd;
629		- const char *basename;
630		- const char *algname;
631		- const char *drvname;
632	959	int err;
633		- int i;
634	960
635	961	err = crypto_register_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
636	962	if (err)
..	..	@@ -640,26 +966,8 @@
640	966	if (err)
641	967	goto unregister_ciphers;
642	968
643		- for (i = 0; i < ARRAY_SIZE(aes_algs); i++) {
644		- if (!(aes_algs[i].base.cra_flags & CRYPTO_ALG_INTERNAL))
645		- continue;
646		-
647		- algname = aes_algs[i].base.cra_name + 2;
648		- drvname = aes_algs[i].base.cra_driver_name + 2;
649		- basename = aes_algs[i].base.cra_driver_name;
650		- simd = simd_skcipher_create_compat(algname, drvname, basename);
651		- err = PTR_ERR(simd);
652		- if (IS_ERR(simd))
653		- goto unregister_simds;
654		-
655		- aes_simd_algs[i] = simd;
656		- }
657		-
658	969	return 0;
659	970
660		-unregister_simds:
661		- aes_exit();
662		- return err;
663	971	unregister_ciphers:
664	972	crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
665	973	return err;
..	..	@@ -671,5 +979,7 @@
671	979	module_init(aes_init);
672	980	EXPORT_SYMBOL(neon_aes_ecb_encrypt);
673	981	EXPORT_SYMBOL(neon_aes_cbc_encrypt);
	982	+EXPORT_SYMBOL(neon_aes_xts_encrypt);
	983	+EXPORT_SYMBOL(neon_aes_xts_decrypt);
674	984	#endif
675	985	module_exit(aes_exit);