forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2023-12-11 072de836f53be56a70cecf70b43ae43b7ce17376 
 kernel/arch/arm64/crypto/ghash-ce-glue.c
....@@ -1,11 +1,8 @@
1
+// SPDX-License-Identifier: GPL-2.0-only
12 /*
23  * Accelerated GHASH implementation with ARMv8 PMULL instructions.
34  *
45  * Copyright (C) 2014 - 2018 Linaro Ltd. <ard.biesheuvel@linaro.org>
5
- *
6
- * This program is free software; you can redistribute it and/or modify it
7
- * under the terms of the GNU General Public License version 2 as published
8
- * by the Free Software Foundation.
96  */
107 
118 #include <asm/neon.h>
....@@ -17,6 +14,7 @@
1714 #include <crypto/gf128mul.h>
1815 #include <crypto/internal/aead.h>
1916 #include <crypto/internal/hash.h>
17
+#include <crypto/internal/simd.h>
2018 #include <crypto/internal/skcipher.h>
2119 #include <crypto/scatterwalk.h>
2220 #include <linux/cpufeature.h>
....@@ -33,12 +31,8 @@
3331 #define GCM_IV_SIZE		12
3432 
3533 struct ghash_key {
36
-	u64			h[2];
37
-	u64			h2[2];
38
-	u64			h3[2];
39
-	u64			h4[2];
40
-
4134 	be128			k;
35
+	u64			h[][2];
4236 };
4337 
4438 struct ghash_desc_ctx {
....@@ -53,46 +47,18 @@
5347 };
5448 
5549 asmlinkage void pmull_ghash_update_p64(int blocks, u64 dg[], const char *src,
56
-				       struct ghash_key const *k,
57
-				       const char *head);
50
+				       u64 const h[][2], const char *head);
5851 
5952 asmlinkage void pmull_ghash_update_p8(int blocks, u64 dg[], const char *src,
60
-				      struct ghash_key const *k,
61
-				      const char *head);
53
+				      u64 const h[][2], const char *head);
6254 
63
-#ifdef CONFIG_CFI_CLANG
64
-static inline void __cfi_pmull_ghash_update_p64(int blocks, u64 dg[],
65
-                const char *src, struct ghash_key const *k, const char *head)
66
-{
67
-        return pmull_ghash_update_p64(blocks, dg, src, k, head);
68
-}
69
-#define pmull_ghash_update_p64 __cfi_pmull_ghash_update_p64
70
-
71
-static inline void __cfi_pmull_ghash_update_p8(int blocks, u64 dg[],
72
-                const char *src, struct ghash_key const *k, const char *head)
73
-{
74
-        return pmull_ghash_update_p8(blocks, dg, src, k, head);
75
-}
76
-#define pmull_ghash_update_p8 __cfi_pmull_ghash_update_p8
77
-#endif
78
-
79
-static void (*pmull_ghash_update)(int blocks, u64 dg[], const char *src,
80
-				  struct ghash_key const *k,
81
-				  const char *head);
82
-
83
-asmlinkage void pmull_gcm_encrypt(int blocks, u64 dg[], u8 dst[],
84
-				  const u8 src[], struct ghash_key const *k,
85
-				  u8 ctr[], u32 const rk[], int rounds,
86
-				  u8 ks[]);
87
-
88
-asmlinkage void pmull_gcm_decrypt(int blocks, u64 dg[], u8 dst[],
89
-				  const u8 src[], struct ghash_key const *k,
90
-				  u8 ctr[], u32 const rk[], int rounds);
91
-
92
-asmlinkage void pmull_gcm_encrypt_block(u8 dst[], u8 const src[],
93
-					u32 const rk[], int rounds);
94
-
95
-asmlinkage void __aes_arm64_encrypt(u32 *rk, u8 *out, const u8 *in, int rounds);
55
+asmlinkage void pmull_gcm_encrypt(int bytes, u8 dst[], const u8 src[],
56
+				  u64 const h[][2], u64 dg[], u8 ctr[],
57
+				  u32 const rk[], int rounds, u8 tag[]);
58
+asmlinkage int pmull_gcm_decrypt(int bytes, u8 dst[], const u8 src[],
59
+				 u64 const h[][2], u64 dg[], u8 ctr[],
60
+				 u32 const rk[], int rounds, const u8 l[],
61
+				 const u8 tag[], u64 authsize);
9662 
9763 static int ghash_init(struct shash_desc *desc)
9864 {
....@@ -105,30 +71,41 @@
10571 static void ghash_do_update(int blocks, u64 dg[], const char *src,
10672 			    struct ghash_key *key, const char *head)
10773 {
108
-	if (likely(may_use_simd())) {
74
+	be128 dst = { cpu_to_be64(dg[1]), cpu_to_be64(dg[0]) };
75
+
76
+	do {
77
+		const u8 *in = src;
78
+
79
+		if (head) {
80
+			in = head;
81
+			blocks++;
82
+			head = NULL;
83
+		} else {
84
+			src += GHASH_BLOCK_SIZE;
85
+		}
86
+
87
+		crypto_xor((u8 *)&dst, in, GHASH_BLOCK_SIZE);
88
+		gf128mul_lle(&dst, &key->k);
89
+	} while (--blocks);
90
+
91
+	dg[0] = be64_to_cpu(dst.b);
92
+	dg[1] = be64_to_cpu(dst.a);
93
+}
94
+
95
+static __always_inline
96
+void ghash_do_simd_update(int blocks, u64 dg[], const char *src,
97
+			  struct ghash_key *key, const char *head,
98
+			  void (*simd_update)(int blocks, u64 dg[],
99
+					      const char *src,
100
+					      u64 const h[][2],
101
+					      const char *head))
102
+{
103
+	if (likely(crypto_simd_usable())) {
109104 		kernel_neon_begin();
110
-		pmull_ghash_update(blocks, dg, src, key, head);
105
+		simd_update(blocks, dg, src, key->h, head);
111106 		kernel_neon_end();
112107 	} else {
113
-		be128 dst = { cpu_to_be64(dg[1]), cpu_to_be64(dg[0]) };
114
-
115
-		do {
116
-			const u8 *in = src;
117
-
118
-			if (head) {
119
-				in = head;
120
-				blocks++;
121
-				head = NULL;
122
-			} else {
123
-				src += GHASH_BLOCK_SIZE;
124
-			}
125
-
126
-			crypto_xor((u8 *)&dst, in, GHASH_BLOCK_SIZE);
127
-			gf128mul_lle(&dst, &key->k);
128
-		} while (--blocks);
129
-
130
-		dg[0] = be64_to_cpu(dst.b);
131
-		dg[1] = be64_to_cpu(dst.a);
108
+		ghash_do_update(blocks, dg, src, key, head);
132109 	}
133110 }
134111 
....@@ -161,8 +138,9 @@
161138 		do {
162139 			int chunk = min(blocks, MAX_BLOCKS);
163140 
164
-			ghash_do_update(chunk, ctx->digest, src, key,
165
-					partial ? ctx->buf : NULL);
141
+			ghash_do_simd_update(chunk, ctx->digest, src, key,
142
+					     partial ? ctx->buf : NULL,
143
+					     pmull_ghash_update_p8);
166144 
167145 			blocks -= chunk;
168146 			src += chunk * GHASH_BLOCK_SIZE;
....@@ -184,7 +162,8 @@
184162 
185163 		memset(ctx->buf + partial, 0, GHASH_BLOCK_SIZE - partial);
186164 
187
-		ghash_do_update(1, ctx->digest, ctx->buf, key, NULL);
165
+		ghash_do_simd_update(1, ctx->digest, ctx->buf, key, NULL,
166
+				     pmull_ghash_update_p8);
188167 	}
189168 	put_unaligned_be64(ctx->digest[1], dst);
190169 	put_unaligned_be64(ctx->digest[0], dst + 8);
....@@ -204,48 +183,27 @@
204183 		h[1] ^= 0xc200000000000000UL;
205184 }
206185 
207
-static int __ghash_setkey(struct ghash_key *key,
208
-			  const u8 *inkey, unsigned int keylen)
209
-{
210
-	be128 h;
211
-
212
-	/* needed for the fallback */
213
-	memcpy(&key->k, inkey, GHASH_BLOCK_SIZE);
214
-
215
-	ghash_reflect(key->h, &key->k);
216
-
217
-	h = key->k;
218
-	gf128mul_lle(&h, &key->k);
219
-	ghash_reflect(key->h2, &h);
220
-
221
-	gf128mul_lle(&h, &key->k);
222
-	ghash_reflect(key->h3, &h);
223
-
224
-	gf128mul_lle(&h, &key->k);
225
-	ghash_reflect(key->h4, &h);
226
-
227
-	return 0;
228
-}
229
-
230186 static int ghash_setkey(struct crypto_shash *tfm,
231187 			const u8 *inkey, unsigned int keylen)
232188 {
233189 	struct ghash_key *key = crypto_shash_ctx(tfm);
234190 
235
-	if (keylen != GHASH_BLOCK_SIZE) {
236
-		crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
191
+	if (keylen != GHASH_BLOCK_SIZE)
237192 		return -EINVAL;
238
-	}
239193 
240
-	return __ghash_setkey(key, inkey, keylen);
194
+	/* needed for the fallback */
195
+	memcpy(&key->k, inkey, GHASH_BLOCK_SIZE);
196
+
197
+	ghash_reflect(key->h[0], &key->k);
198
+	return 0;
241199 }
242200 
243201 static struct shash_alg ghash_alg = {
244202 	.base.cra_name		= "ghash",
245
-	.base.cra_driver_name	= "ghash-ce",
246
-	.base.cra_priority	= 200,
203
+	.base.cra_driver_name	= "ghash-neon",
204
+	.base.cra_priority	= 150,
247205 	.base.cra_blocksize	= GHASH_BLOCK_SIZE,
248
-	.base.cra_ctxsize	= sizeof(struct ghash_key),
206
+	.base.cra_ctxsize	= sizeof(struct ghash_key) + sizeof(u64[2]),
249207 	.base.cra_module	= THIS_MODULE,
250208 
251209 	.digestsize		= GHASH_DIGEST_SIZE,
....@@ -273,18 +231,31 @@
273231 {
274232 	struct gcm_aes_ctx *ctx = crypto_aead_ctx(tfm);
275233 	u8 key[GHASH_BLOCK_SIZE];
234
+	be128 h;
276235 	int ret;
277236 
278
-	ret = crypto_aes_expand_key(&ctx->aes_key, inkey, keylen);
279
-	if (ret) {
280
-		tfm->base.crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
237
+	ret = aes_expandkey(&ctx->aes_key, inkey, keylen);
238
+	if (ret)
281239 		return -EINVAL;
282
-	}
283240 
284
-	__aes_arm64_encrypt(ctx->aes_key.key_enc, key, (u8[AES_BLOCK_SIZE]){},
285
-			    num_rounds(&ctx->aes_key));
241
+	aes_encrypt(&ctx->aes_key, key, (u8[AES_BLOCK_SIZE]){});
286242 
287
-	return __ghash_setkey(&ctx->ghash_key, key, sizeof(be128));
243
+	/* needed for the fallback */
244
+	memcpy(&ctx->ghash_key.k, key, GHASH_BLOCK_SIZE);
245
+
246
+	ghash_reflect(ctx->ghash_key.h[0], &ctx->ghash_key.k);
247
+
248
+	h = ctx->ghash_key.k;
249
+	gf128mul_lle(&h, &ctx->ghash_key.k);
250
+	ghash_reflect(ctx->ghash_key.h[1], &h);
251
+
252
+	gf128mul_lle(&h, &ctx->ghash_key.k);
253
+	ghash_reflect(ctx->ghash_key.h[2], &h);
254
+
255
+	gf128mul_lle(&h, &ctx->ghash_key.k);
256
+	ghash_reflect(ctx->ghash_key.h[3], &h);
257
+
258
+	return 0;
288259 }
289260 
290261 static int gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
....@@ -316,8 +287,9 @@
316287 	if (count >= GHASH_BLOCK_SIZE || *buf_count == GHASH_BLOCK_SIZE) {
317288 		int blocks = count / GHASH_BLOCK_SIZE;
318289 
319
-		ghash_do_update(blocks, dg, src, &ctx->ghash_key,
320
-				*buf_count ? buf : NULL);
290
+		ghash_do_simd_update(blocks, dg, src, &ctx->ghash_key,
291
+				     *buf_count ? buf : NULL,
292
+				     pmull_ghash_update_p64);
321293 
322294 		src += blocks * GHASH_BLOCK_SIZE;
323295 		count %= GHASH_BLOCK_SIZE;
....@@ -361,141 +333,116 @@
361333 
362334 	if (buf_count) {
363335 		memset(&buf[buf_count], 0, GHASH_BLOCK_SIZE - buf_count);
364
-		ghash_do_update(1, dg, buf, &ctx->ghash_key, NULL);
336
+		ghash_do_simd_update(1, dg, buf, &ctx->ghash_key, NULL,
337
+				     pmull_ghash_update_p64);
365338 	}
366
-}
367
-
368
-static void gcm_final(struct aead_request *req, struct gcm_aes_ctx *ctx,
369
-		      u64 dg[], u8 tag[], int cryptlen)
370
-{
371
-	u8 mac[AES_BLOCK_SIZE];
372
-	u128 lengths;
373
-
374
-	lengths.a = cpu_to_be64(req->assoclen * 8);
375
-	lengths.b = cpu_to_be64(cryptlen * 8);
376
-
377
-	ghash_do_update(1, dg, (void *)&lengths, &ctx->ghash_key, NULL);
378
-
379
-	put_unaligned_be64(dg[1], mac);
380
-	put_unaligned_be64(dg[0], mac + 8);
381
-
382
-	crypto_xor(tag, mac, AES_BLOCK_SIZE);
383339 }
384340 
385341 static int gcm_encrypt(struct aead_request *req)
386342 {
387343 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
388344 	struct gcm_aes_ctx *ctx = crypto_aead_ctx(aead);
389
-	struct skcipher_walk walk;
390
-	u8 iv[AES_BLOCK_SIZE];
391
-	u8 ks[2 * AES_BLOCK_SIZE];
392
-	u8 tag[AES_BLOCK_SIZE];
393
-	u64 dg[2] = {};
394345 	int nrounds = num_rounds(&ctx->aes_key);
346
+	struct skcipher_walk walk;
347
+	u8 buf[AES_BLOCK_SIZE];
348
+	u8 iv[AES_BLOCK_SIZE];
349
+	u64 dg[2] = {};
350
+	be128 lengths;
351
+	u8 *tag;
395352 	int err;
353
+
354
+	lengths.a = cpu_to_be64(req->assoclen * 8);
355
+	lengths.b = cpu_to_be64(req->cryptlen * 8);
396356 
397357 	if (req->assoclen)
398358 		gcm_calculate_auth_mac(req, dg);
399359 
400360 	memcpy(iv, req->iv, GCM_IV_SIZE);
401
-	put_unaligned_be32(1, iv + GCM_IV_SIZE);
361
+	put_unaligned_be32(2, iv + GCM_IV_SIZE);
402362 
403363 	err = skcipher_walk_aead_encrypt(&walk, req, false);
404364 
405
-	if (likely(may_use_simd() && walk.total >= 2 * AES_BLOCK_SIZE)) {
406
-		u32 const *rk = NULL;
407
-
408
-		kernel_neon_begin();
409
-		pmull_gcm_encrypt_block(tag, iv, ctx->aes_key.key_enc, nrounds);
410
-		put_unaligned_be32(2, iv + GCM_IV_SIZE);
411
-		pmull_gcm_encrypt_block(ks, iv, NULL, nrounds);
412
-		put_unaligned_be32(3, iv + GCM_IV_SIZE);
413
-		pmull_gcm_encrypt_block(ks + AES_BLOCK_SIZE, iv, NULL, nrounds);
414
-		put_unaligned_be32(4, iv + GCM_IV_SIZE);
415
-
365
+	if (likely(crypto_simd_usable())) {
416366 		do {
417
-			int blocks = walk.nbytes / (2 * AES_BLOCK_SIZE) * 2;
367
+			const u8 *src = walk.src.virt.addr;
368
+			u8 *dst = walk.dst.virt.addr;
369
+			int nbytes = walk.nbytes;
418370 
419
-			if (rk)
420
-				kernel_neon_begin();
371
+			tag = (u8 *)&lengths;
421372 
422
-			pmull_gcm_encrypt(blocks, dg, walk.dst.virt.addr,
423
-					  walk.src.virt.addr, &ctx->ghash_key,
424
-					  iv, rk, nrounds, ks);
373
+			if (unlikely(nbytes > 0 && nbytes < AES_BLOCK_SIZE)) {
374
+				src = dst = memcpy(buf + sizeof(buf) - nbytes,
375
+						   src, nbytes);
376
+			} else if (nbytes < walk.total) {
377
+				nbytes &= ~(AES_BLOCK_SIZE - 1);
378
+				tag = NULL;
379
+			}
380
+
381
+			kernel_neon_begin();
382
+			pmull_gcm_encrypt(nbytes, dst, src, ctx->ghash_key.h,
383
+					  dg, iv, ctx->aes_key.key_enc, nrounds,
384
+					  tag);
425385 			kernel_neon_end();
426386 
427
-			err = skcipher_walk_done(&walk,
428
-					walk.nbytes % (2 * AES_BLOCK_SIZE));
387
+			if (unlikely(!nbytes))
388
+				break;
429389 
430
-			rk = ctx->aes_key.key_enc;
431
-		} while (walk.nbytes >= 2 * AES_BLOCK_SIZE);
390
+			if (unlikely(nbytes > 0 && nbytes < AES_BLOCK_SIZE))
391
+				memcpy(walk.dst.virt.addr,
392
+				       buf + sizeof(buf) - nbytes, nbytes);
393
+
394
+			err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
395
+		} while (walk.nbytes);
432396 	} else {
433
-		__aes_arm64_encrypt(ctx->aes_key.key_enc, tag, iv, nrounds);
434
-		put_unaligned_be32(2, iv + GCM_IV_SIZE);
435
-
436
-		while (walk.nbytes >= (2 * AES_BLOCK_SIZE)) {
437
-			const int blocks =
438
-				walk.nbytes / (2 * AES_BLOCK_SIZE) * 2;
397
+		while (walk.nbytes >= AES_BLOCK_SIZE) {
398
+			int blocks = walk.nbytes / AES_BLOCK_SIZE;
399
+			const u8 *src = walk.src.virt.addr;
439400 			u8 *dst = walk.dst.virt.addr;
440
-			u8 *src = walk.src.virt.addr;
441401 			int remaining = blocks;
442402 
443403 			do {
444
-				__aes_arm64_encrypt(ctx->aes_key.key_enc,
445
-						    ks, iv, nrounds);
446
-				crypto_xor_cpy(dst, src, ks, AES_BLOCK_SIZE);
404
+				aes_encrypt(&ctx->aes_key, buf, iv);
405
+				crypto_xor_cpy(dst, src, buf, AES_BLOCK_SIZE);
447406 				crypto_inc(iv, AES_BLOCK_SIZE);
448407 
449408 				dst += AES_BLOCK_SIZE;
450409 				src += AES_BLOCK_SIZE;
451410 			} while (--remaining > 0);
452411 
453
-			ghash_do_update(blocks, dg,
454
-					walk.dst.virt.addr, &ctx->ghash_key,
455
-					NULL);
412
+			ghash_do_update(blocks, dg, walk.dst.virt.addr,
413
+					&ctx->ghash_key, NULL);
456414 
457415 			err = skcipher_walk_done(&walk,
458
-						 walk.nbytes % (2 * AES_BLOCK_SIZE));
416
+						 walk.nbytes % AES_BLOCK_SIZE);
459417 		}
418
+
419
+		/* handle the tail */
460420 		if (walk.nbytes) {
461
-			__aes_arm64_encrypt(ctx->aes_key.key_enc, ks, iv,
462
-					    nrounds);
463
-			if (walk.nbytes > AES_BLOCK_SIZE) {
464
-				crypto_inc(iv, AES_BLOCK_SIZE);
465
-				__aes_arm64_encrypt(ctx->aes_key.key_enc,
466
-					            ks + AES_BLOCK_SIZE, iv,
467
-						    nrounds);
468
-			}
469
-		}
470
-	}
421
+			aes_encrypt(&ctx->aes_key, buf, iv);
471422 
472
-	/* handle the tail */
473
-	if (walk.nbytes) {
474
-		u8 buf[GHASH_BLOCK_SIZE];
475
-		unsigned int nbytes = walk.nbytes;
476
-		u8 *dst = walk.dst.virt.addr;
477
-		u8 *head = NULL;
423
+			crypto_xor_cpy(walk.dst.virt.addr, walk.src.virt.addr,
424
+				       buf, walk.nbytes);
478425 
479
-		crypto_xor_cpy(walk.dst.virt.addr, walk.src.virt.addr, ks,
480
-			       walk.nbytes);
481
-
482
-		if (walk.nbytes > GHASH_BLOCK_SIZE) {
483
-			head = dst;
484
-			dst += GHASH_BLOCK_SIZE;
485
-			nbytes %= GHASH_BLOCK_SIZE;
426
+			memcpy(buf, walk.dst.virt.addr, walk.nbytes);
427
+			memset(buf + walk.nbytes, 0, sizeof(buf) - walk.nbytes);
486428 		}
487429 
488
-		memcpy(buf, dst, nbytes);
489
-		memset(buf + nbytes, 0, GHASH_BLOCK_SIZE - nbytes);
490
-		ghash_do_update(!!nbytes, dg, buf, &ctx->ghash_key, head);
430
+		tag = (u8 *)&lengths;
431
+		ghash_do_update(1, dg, tag, &ctx->ghash_key,
432
+				walk.nbytes ? buf : NULL);
491433 
492
-		err = skcipher_walk_done(&walk, 0);
434
+		if (walk.nbytes)
435
+			err = skcipher_walk_done(&walk, 0);
436
+
437
+		put_unaligned_be64(dg[1], tag);
438
+		put_unaligned_be64(dg[0], tag + 8);
439
+		put_unaligned_be32(1, iv + GCM_IV_SIZE);
440
+		aes_encrypt(&ctx->aes_key, iv, iv);
441
+		crypto_xor(tag, iv, AES_BLOCK_SIZE);
493442 	}
494443 
495444 	if (err)
496445 		return err;
497
-
498
-	gcm_final(req, ctx, dg, tag, req->cryptlen);
499446 
500447 	/* copy authtag to end of dst */
501448 	scatterwalk_map_and_copy(tag, req->dst, req->assoclen + req->cryptlen,
....@@ -509,78 +456,81 @@
509456 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
510457 	struct gcm_aes_ctx *ctx = crypto_aead_ctx(aead);
511458 	unsigned int authsize = crypto_aead_authsize(aead);
512
-	struct skcipher_walk walk;
513
-	u8 iv[2 * AES_BLOCK_SIZE];
514
-	u8 tag[AES_BLOCK_SIZE];
515
-	u8 buf[2 * GHASH_BLOCK_SIZE];
516
-	u64 dg[2] = {};
517459 	int nrounds = num_rounds(&ctx->aes_key);
460
+	struct skcipher_walk walk;
461
+	u8 otag[AES_BLOCK_SIZE];
462
+	u8 buf[AES_BLOCK_SIZE];
463
+	u8 iv[AES_BLOCK_SIZE];
464
+	u64 dg[2] = {};
465
+	be128 lengths;
466
+	u8 *tag;
518467 	int err;
468
+
469
+	lengths.a = cpu_to_be64(req->assoclen * 8);
470
+	lengths.b = cpu_to_be64((req->cryptlen - authsize) * 8);
519471 
520472 	if (req->assoclen)
521473 		gcm_calculate_auth_mac(req, dg);
522474 
523475 	memcpy(iv, req->iv, GCM_IV_SIZE);
524
-	put_unaligned_be32(1, iv + GCM_IV_SIZE);
476
+	put_unaligned_be32(2, iv + GCM_IV_SIZE);
477
+
478
+	scatterwalk_map_and_copy(otag, req->src,
479
+				 req->assoclen + req->cryptlen - authsize,
480
+				 authsize, 0);
525481 
526482 	err = skcipher_walk_aead_decrypt(&walk, req, false);
527483 
528
-	if (likely(may_use_simd() && walk.total >= 2 * AES_BLOCK_SIZE)) {
529
-		u32 const *rk = NULL;
530
-
531
-		kernel_neon_begin();
532
-		pmull_gcm_encrypt_block(tag, iv, ctx->aes_key.key_enc, nrounds);
533
-		put_unaligned_be32(2, iv + GCM_IV_SIZE);
484
+	if (likely(crypto_simd_usable())) {
485
+		int ret;
534486 
535487 		do {
536
-			int blocks = walk.nbytes / (2 * AES_BLOCK_SIZE) * 2;
537
-			int rem = walk.total - blocks * AES_BLOCK_SIZE;
488
+			const u8 *src = walk.src.virt.addr;
489
+			u8 *dst = walk.dst.virt.addr;
490
+			int nbytes = walk.nbytes;
538491 
539
-			if (rk)
540
-				kernel_neon_begin();
492
+			tag = (u8 *)&lengths;
541493 
542
-			pmull_gcm_decrypt(blocks, dg, walk.dst.virt.addr,
543
-					  walk.src.virt.addr, &ctx->ghash_key,
544
-					  iv, rk, nrounds);
545
-
546
-			/* check if this is the final iteration of the loop */
547
-			if (rem < (2 * AES_BLOCK_SIZE)) {
548
-				u8 *iv2 = iv + AES_BLOCK_SIZE;
549
-
550
-				if (rem > AES_BLOCK_SIZE) {
551
-					memcpy(iv2, iv, AES_BLOCK_SIZE);
552
-					crypto_inc(iv2, AES_BLOCK_SIZE);
553
-				}
554
-
555
-				pmull_gcm_encrypt_block(iv, iv, NULL, nrounds);
556
-
557
-				if (rem > AES_BLOCK_SIZE)
558
-					pmull_gcm_encrypt_block(iv2, iv2, NULL,
559
-								nrounds);
494
+			if (unlikely(nbytes > 0 && nbytes < AES_BLOCK_SIZE)) {
495
+				src = dst = memcpy(buf + sizeof(buf) - nbytes,
496
+						   src, nbytes);
497
+			} else if (nbytes < walk.total) {
498
+				nbytes &= ~(AES_BLOCK_SIZE - 1);
499
+				tag = NULL;
560500 			}
561501 
502
+			kernel_neon_begin();
503
+			ret = pmull_gcm_decrypt(nbytes, dst, src,
504
+						ctx->ghash_key.h,
505
+						dg, iv, ctx->aes_key.key_enc,
506
+						nrounds, tag, otag, authsize);
562507 			kernel_neon_end();
563508 
564
-			err = skcipher_walk_done(&walk,
565
-					walk.nbytes % (2 * AES_BLOCK_SIZE));
509
+			if (unlikely(!nbytes))
510
+				break;
566511 
567
-			rk = ctx->aes_key.key_enc;
568
-		} while (walk.nbytes >= 2 * AES_BLOCK_SIZE);
512
+			if (unlikely(nbytes > 0 && nbytes < AES_BLOCK_SIZE))
513
+				memcpy(walk.dst.virt.addr,
514
+				       buf + sizeof(buf) - nbytes, nbytes);
515
+
516
+			err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
517
+		} while (walk.nbytes);
518
+
519
+		if (err)
520
+			return err;
521
+		if (ret)
522
+			return -EBADMSG;
569523 	} else {
570
-		__aes_arm64_encrypt(ctx->aes_key.key_enc, tag, iv, nrounds);
571
-		put_unaligned_be32(2, iv + GCM_IV_SIZE);
572
-
573
-		while (walk.nbytes >= (2 * AES_BLOCK_SIZE)) {
574
-			int blocks = walk.nbytes / (2 * AES_BLOCK_SIZE) * 2;
524
+		while (walk.nbytes >= AES_BLOCK_SIZE) {
525
+			int blocks = walk.nbytes / AES_BLOCK_SIZE;
526
+			const u8 *src = walk.src.virt.addr;
575527 			u8 *dst = walk.dst.virt.addr;
576
-			u8 *src = walk.src.virt.addr;
577528 
578529 			ghash_do_update(blocks, dg, walk.src.virt.addr,
579530 					&ctx->ghash_key, NULL);
580531 
581532 			do {
582
-				__aes_arm64_encrypt(ctx->aes_key.key_enc,
583
-						    buf, iv, nrounds);
533
+				aes_encrypt(&ctx->aes_key, buf, iv);
584534 				crypto_xor_cpy(dst, src, buf, AES_BLOCK_SIZE);
585535 				crypto_inc(iv, AES_BLOCK_SIZE);
586536 
....@@ -589,63 +539,48 @@
589539 			} while (--blocks > 0);
590540 
591541 			err = skcipher_walk_done(&walk,
592
-						 walk.nbytes % (2 * AES_BLOCK_SIZE));
542
+						 walk.nbytes % AES_BLOCK_SIZE);
593543 		}
544
+
545
+		/* handle the tail */
594546 		if (walk.nbytes) {
595
-			if (walk.nbytes > AES_BLOCK_SIZE) {
596
-				u8 *iv2 = iv + AES_BLOCK_SIZE;
597
-
598
-				memcpy(iv2, iv, AES_BLOCK_SIZE);
599
-				crypto_inc(iv2, AES_BLOCK_SIZE);
600
-
601
-				__aes_arm64_encrypt(ctx->aes_key.key_enc, iv2,
602
-						    iv2, nrounds);
603
-			}
604
-			__aes_arm64_encrypt(ctx->aes_key.key_enc, iv, iv,
605
-					    nrounds);
606
-		}
607
-	}
608
-
609
-	/* handle the tail */
610
-	if (walk.nbytes) {
611
-		const u8 *src = walk.src.virt.addr;
612
-		const u8 *head = NULL;
613
-		unsigned int nbytes = walk.nbytes;
614
-
615
-		if (walk.nbytes > GHASH_BLOCK_SIZE) {
616
-			head = src;
617
-			src += GHASH_BLOCK_SIZE;
618
-			nbytes %= GHASH_BLOCK_SIZE;
547
+			memcpy(buf, walk.src.virt.addr, walk.nbytes);
548
+			memset(buf + walk.nbytes, 0, sizeof(buf) - walk.nbytes);
619549 		}
620550 
621
-		memcpy(buf, src, nbytes);
622
-		memset(buf + nbytes, 0, GHASH_BLOCK_SIZE - nbytes);
623
-		ghash_do_update(!!nbytes, dg, buf, &ctx->ghash_key, head);
551
+		tag = (u8 *)&lengths;
552
+		ghash_do_update(1, dg, tag, &ctx->ghash_key,
553
+				walk.nbytes ? buf : NULL);
624554 
625
-		crypto_xor_cpy(walk.dst.virt.addr, walk.src.virt.addr, iv,
626
-			       walk.nbytes);
555
+		if (walk.nbytes) {
556
+			aes_encrypt(&ctx->aes_key, buf, iv);
627557 
628
-		err = skcipher_walk_done(&walk, 0);
558
+			crypto_xor_cpy(walk.dst.virt.addr, walk.src.virt.addr,
559
+				       buf, walk.nbytes);
560
+
561
+			err = skcipher_walk_done(&walk, 0);
562
+		}
563
+
564
+		if (err)
565
+			return err;
566
+
567
+		put_unaligned_be64(dg[1], tag);
568
+		put_unaligned_be64(dg[0], tag + 8);
569
+		put_unaligned_be32(1, iv + GCM_IV_SIZE);
570
+		aes_encrypt(&ctx->aes_key, iv, iv);
571
+		crypto_xor(tag, iv, AES_BLOCK_SIZE);
572
+
573
+		if (crypto_memneq(tag, otag, authsize)) {
574
+			memzero_explicit(tag, AES_BLOCK_SIZE);
575
+			return -EBADMSG;
576
+		}
629577 	}
630
-
631
-	if (err)
632
-		return err;
633
-
634
-	gcm_final(req, ctx, dg, tag, req->cryptlen - authsize);
635
-
636
-	/* compare calculated auth tag with the stored one */
637
-	scatterwalk_map_and_copy(buf, req->src,
638
-				 req->assoclen + req->cryptlen - authsize,
639
-				 authsize, 0);
640
-
641
-	if (crypto_memneq(tag, buf, authsize))
642
-		return -EBADMSG;
643578 	return 0;
644579 }
645580 
646581 static struct aead_alg gcm_aes_alg = {
647582 	.ivsize			= GCM_IV_SIZE,
648
-	.chunksize		= 2 * AES_BLOCK_SIZE,
583
+	.chunksize		= AES_BLOCK_SIZE,
649584 	.maxauthsize		= AES_BLOCK_SIZE,
650585 	.setkey			= gcm_setkey,
651586 	.setauthsize		= gcm_setauthsize,
....@@ -656,39 +591,28 @@
656591 	.base.cra_driver_name	= "gcm-aes-ce",
657592 	.base.cra_priority	= 300,
658593 	.base.cra_blocksize	= 1,
659
-	.base.cra_ctxsize	= sizeof(struct gcm_aes_ctx),
594
+	.base.cra_ctxsize	= sizeof(struct gcm_aes_ctx) +
595
+				  4 * sizeof(u64[2]),
660596 	.base.cra_module	= THIS_MODULE,
661597 };
662598 
663599 static int __init ghash_ce_mod_init(void)
664600 {
665
-	int ret;
666
-
667
-	if (!(elf_hwcap & HWCAP_ASIMD))
601
+	if (!cpu_have_named_feature(ASIMD))
668602 		return -ENODEV;
669603 
670
-	if (elf_hwcap & HWCAP_PMULL)
671
-		pmull_ghash_update = pmull_ghash_update_p64;
604
+	if (cpu_have_named_feature(PMULL))
605
+		return crypto_register_aead(&gcm_aes_alg);
672606 
673
-	else
674
-		pmull_ghash_update = pmull_ghash_update_p8;
675
-
676
-	ret = crypto_register_shash(&ghash_alg);
677
-	if (ret)
678
-		return ret;
679
-
680
-	if (elf_hwcap & HWCAP_PMULL) {
681
-		ret = crypto_register_aead(&gcm_aes_alg);
682
-		if (ret)
683
-			crypto_unregister_shash(&ghash_alg);
684
-	}
685
-	return ret;
607
+	return crypto_register_shash(&ghash_alg);
686608 }
687609 
688610 static void __exit ghash_ce_mod_exit(void)
689611 {
690
-	crypto_unregister_shash(&ghash_alg);
691
-	crypto_unregister_aead(&gcm_aes_alg);
612
+	if (cpu_have_named_feature(PMULL))
613
+		crypto_unregister_aead(&gcm_aes_alg);
614
+	else
615
+		crypto_unregister_shash(&ghash_alg);
692616 }
693617 
694618 static const struct cpu_feature ghash_cpu_feature[] = {