write in 30M

hc

2024-02-20 ea08eeccae9297f7aabd2ef7f0c2517ac4549acc

 kernel/arch/arm/crypto/ghash-ce-glue.c
..
..
@@ -1,25 +1,25 @@
1
+// SPDX-License-Identifier: GPL-2.0-only
1
2
 /*
2
3
  * Accelerated GHASH implementation with ARMv8 vmull.p64 instructions.
3
4
  *
4
- * Copyright (C) 2015 Linaro Ltd. <ard.biesheuvel@linaro.org>
5
- *
6
- * This program is free software; you can redistribute it and/or modify it
7
- * under the terms of the GNU General Public License version 2 as published
8
- * by the Free Software Foundation.
5
+ * Copyright (C) 2015 - 2018 Linaro Ltd. <ard.biesheuvel@linaro.org>
9
6
  */
10
7
 
11
8
 #include <asm/hwcap.h>
12
9
 #include <asm/neon.h>
13
10
 #include <asm/simd.h>
14
11
 #include <asm/unaligned.h>
12
+#include <crypto/b128ops.h>
15
13
 #include <crypto/cryptd.h>
16
14
 #include <crypto/internal/hash.h>
15
+#include <crypto/internal/simd.h>
17
16
 #include <crypto/gf128mul.h>
18
17
 #include <linux/cpufeature.h>
19
18
 #include <linux/crypto.h>
19
+#include <linux/jump_label.h>
20
20
 #include <linux/module.h>
21
21
 
22
-MODULE_DESCRIPTION("GHASH secure hash using ARMv8 Crypto Extensions");
22
+MODULE_DESCRIPTION("GHASH hash function using ARMv8 Crypto Extensions");
23
23
 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
24
24
 MODULE_LICENSE("GPL v2");
25
25
 MODULE_ALIAS_CRYPTO("ghash");
..
..
@@ -28,8 +28,8 @@
28
28
 #define GHASH_DIGEST_SIZE	16
29
29
 
30
30
 struct ghash_key {
31
-	u64	a;
32
-	u64	b;
31
+	be128	k;
32
+	u64	h[][2];
33
33
 };
34
34
 
35
35
 struct ghash_desc_ctx {
..
..
@@ -43,16 +43,12 @@
43
43
 };
44
44
 
45
45
 asmlinkage void pmull_ghash_update_p64(int blocks, u64 dg[], const char *src,
46
-				       struct ghash_key const *k,
47
-				       const char *head);
46
+				       u64 const h[][2], const char *head);
48
47
 
49
48
 asmlinkage void pmull_ghash_update_p8(int blocks, u64 dg[], const char *src,
50
-				      struct ghash_key const *k,
51
-				      const char *head);
49
+				      u64 const h[][2], const char *head);
52
50
 
53
-static void (*pmull_ghash_update)(int blocks, u64 dg[], const char *src,
54
-				  struct ghash_key const *k,
55
-				  const char *head);
51
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(use_p64);
56
52
 
57
53
 static int ghash_init(struct shash_desc *desc)
58
54
 {
..
..
@@ -60,6 +56,39 @@
60
56
 
61
57
 	*ctx = (struct ghash_desc_ctx){};
62
58
 	return 0;
59
+}
60
+
61
+static void ghash_do_update(int blocks, u64 dg[], const char *src,
62
+			    struct ghash_key *key, const char *head)
63
+{
64
+	if (likely(crypto_simd_usable())) {
65
+		kernel_neon_begin();
66
+		if (static_branch_likely(&use_p64))
67
+			pmull_ghash_update_p64(blocks, dg, src, key->h, head);
68
+		else
69
+			pmull_ghash_update_p8(blocks, dg, src, key->h, head);
70
+		kernel_neon_end();
71
+	} else {
72
+		be128 dst = { cpu_to_be64(dg[1]), cpu_to_be64(dg[0]) };
73
+
74
+		do {
75
+			const u8 *in = src;
76
+
77
+			if (head) {
78
+				in = head;
79
+				blocks++;
80
+				head = NULL;
81
+			} else {
82
+				src += GHASH_BLOCK_SIZE;
83
+			}
84
+
85
+			crypto_xor((u8 *)&dst, in, GHASH_BLOCK_SIZE);
86
+			gf128mul_lle(&dst, &key->k);
87
+		} while (--blocks);
88
+
89
+		dg[0] = be64_to_cpu(dst.b);
90
+		dg[1] = be64_to_cpu(dst.a);
91
+	}
63
92
 }
64
93
 
65
94
 static int ghash_update(struct shash_desc *desc, const u8 *src,
..
..
@@ -85,10 +114,8 @@
85
114
 		blocks = len / GHASH_BLOCK_SIZE;
86
115
 		len %= GHASH_BLOCK_SIZE;
87
116
 
88
-		kernel_neon_begin();
89
-		pmull_ghash_update(blocks, ctx->digest, src, key,
90
-				   partial ? ctx->buf : NULL);
91
-		kernel_neon_end();
117
+		ghash_do_update(blocks, ctx->digest, src, key,
118
+				partial ? ctx->buf : NULL);
92
119
 		src += blocks * GHASH_BLOCK_SIZE;
93
120
 		partial = 0;
94
121
 	}
..
..
@@ -106,9 +133,7 @@
106
133
 		struct ghash_key *key = crypto_shash_ctx(desc->tfm);
107
134
 
108
135
 		memset(ctx->buf + partial, 0, GHASH_BLOCK_SIZE - partial);
109
-		kernel_neon_begin();
110
-		pmull_ghash_update(1, ctx->digest, ctx->buf, key, NULL);
111
-		kernel_neon_end();
136
+		ghash_do_update(1, ctx->digest, ctx->buf, key, NULL);
112
137
 	}
113
138
 	put_unaligned_be64(ctx->digest[1], dst);
114
139
 	put_unaligned_be64(ctx->digest[0], dst + 8);
..
..
@@ -117,27 +142,41 @@
117
142
 	return 0;
118
143
 }
119
144
 
145
+static void ghash_reflect(u64 h[], const be128 *k)
146
+{
147
+	u64 carry = be64_to_cpu(k->a) >> 63;
148
+
149
+	h[0] = (be64_to_cpu(k->b) << 1) | carry;
150
+	h[1] = (be64_to_cpu(k->a) << 1) | (be64_to_cpu(k->b) >> 63);
151
+
152
+	if (carry)
153
+		h[1] ^= 0xc200000000000000UL;
154
+}
155
+
120
156
 static int ghash_setkey(struct crypto_shash *tfm,
121
157
 			const u8 *inkey, unsigned int keylen)
122
158
 {
123
159
 	struct ghash_key *key = crypto_shash_ctx(tfm);
124
-	u64 a, b;
125
160
 
126
-	if (keylen != GHASH_BLOCK_SIZE) {
127
-		crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
161
+	if (keylen != GHASH_BLOCK_SIZE)
128
162
 		return -EINVAL;
163
+
164
+	/* needed for the fallback */
165
+	memcpy(&key->k, inkey, GHASH_BLOCK_SIZE);
166
+	ghash_reflect(key->h[0], &key->k);
167
+
168
+	if (static_branch_likely(&use_p64)) {
169
+		be128 h = key->k;
170
+
171
+		gf128mul_lle(&h, &key->k);
172
+		ghash_reflect(key->h[1], &h);
173
+
174
+		gf128mul_lle(&h, &key->k);
175
+		ghash_reflect(key->h[2], &h);
176
+
177
+		gf128mul_lle(&h, &key->k);
178
+		ghash_reflect(key->h[3], &h);
129
179
 	}
130
-
131
-	/* perform multiplication by 'x' in GF(2^128) */
132
-	b = get_unaligned_be64(inkey);
133
-	a = get_unaligned_be64(inkey + 8);
134
-
135
-	key->a = (a << 1) | (b >> 63);
136
-	key->b = (b << 1) | (a >> 63);
137
-
138
-	if (b >> 63)
139
-		key->b ^= 0xc200000000000000UL;
140
-
141
180
 	return 0;
142
181
 }
143
182
 
..
..
@@ -148,15 +187,13 @@
148
187
 	.final			= ghash_final,
149
188
 	.setkey			= ghash_setkey,
150
189
 	.descsize		= sizeof(struct ghash_desc_ctx),
151
-	.base			= {
152
-		.cra_name	= "__ghash",
153
-		.cra_driver_name = "__driver-ghash-ce",
154
-		.cra_priority	= 0,
155
-		.cra_flags	= CRYPTO_ALG_INTERNAL,
156
-		.cra_blocksize	= GHASH_BLOCK_SIZE,
157
-		.cra_ctxsize	= sizeof(struct ghash_key),
158
-		.cra_module	= THIS_MODULE,
159
-	},
190
+
191
+	.base.cra_name		= "ghash",
192
+	.base.cra_driver_name	= "ghash-ce-sync",
193
+	.base.cra_priority	= 300 - 1,
194
+	.base.cra_blocksize	= GHASH_BLOCK_SIZE,
195
+	.base.cra_ctxsize	= sizeof(struct ghash_key) + sizeof(u64[2]),
196
+	.base.cra_module	= THIS_MODULE,
160
197
 };
161
198
 
162
199
 static int ghash_async_init(struct ahash_request *req)
..
..
@@ -169,7 +206,6 @@
169
206
 	struct crypto_shash *child = cryptd_ahash_child(cryptd_tfm);
170
207
 
171
208
 	desc->tfm = child;
172
-	desc->flags = req->base.flags;
173
209
 	return crypto_shash_init(desc);
174
210
 }
175
211
 
..
..
@@ -180,7 +216,7 @@
180
216
 	struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
181
217
 	struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
182
218
 
183
-	if (!may_use_simd() ||
219
+	if (!crypto_simd_usable() ||
184
220
 	    (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) {
185
221
 		memcpy(cryptd_req, req, sizeof(*req));
186
222
 		ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
..
..
@@ -198,7 +234,7 @@
198
234
 	struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
199
235
 	struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
200
236
 
201
-	if (!may_use_simd() ||
237
+	if (!crypto_simd_usable() ||
202
238
 	    (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) {
203
239
 		memcpy(cryptd_req, req, sizeof(*req));
204
240
 		ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
..
..
@@ -216,7 +252,7 @@
216
252
 	struct ahash_request *cryptd_req = ahash_request_ctx(req);
217
253
 	struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
218
254
 
219
-	if (!may_use_simd() ||
255
+	if (!crypto_simd_usable() ||
220
256
 	    (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) {
221
257
 		memcpy(cryptd_req, req, sizeof(*req));
222
258
 		ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
..
..
@@ -226,7 +262,6 @@
226
262
 		struct crypto_shash *child = cryptd_ahash_child(cryptd_tfm);
227
263
 
228
264
 		desc->tfm = child;
229
-		desc->flags = req->base.flags;
230
265
 		return shash_ahash_digest(req, desc);
231
266
 	}
232
267
 }
..
..
@@ -239,7 +274,6 @@
239
274
 	struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
240
275
 
241
276
 	desc->tfm = cryptd_ahash_child(ctx->cryptd_tfm);
242
-	desc->flags = req->base.flags;
243
277
 
244
278
 	return crypto_shash_import(desc, in);
245
279
 }
..
..
@@ -257,16 +291,11 @@
257
291
 {
258
292
 	struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
259
293
 	struct crypto_ahash *child = &ctx->cryptd_tfm->base;
260
-	int err;
261
294
 
262
295
 	crypto_ahash_clear_flags(child, CRYPTO_TFM_REQ_MASK);
263
296
 	crypto_ahash_set_flags(child, crypto_ahash_get_flags(tfm)
264
297
 			       & CRYPTO_TFM_REQ_MASK);
265
-	err = crypto_ahash_setkey(child, key, keylen);
266
-	crypto_ahash_set_flags(tfm, crypto_ahash_get_flags(child)
267
-			       & CRYPTO_TFM_RES_MASK);
268
-
269
-	return err;
298
+	return crypto_ahash_setkey(child, key, keylen);
270
299
 }
271
300
 
272
301
 static int ghash_async_init_tfm(struct crypto_tfm *tfm)
..
..
@@ -274,9 +303,7 @@
274
303
 	struct cryptd_ahash *cryptd_tfm;
275
304
 	struct ghash_async_ctx *ctx = crypto_tfm_ctx(tfm);
276
305
 
277
-	cryptd_tfm = cryptd_alloc_ahash("__driver-ghash-ce",
278
-					CRYPTO_ALG_INTERNAL,
279
-					CRYPTO_ALG_INTERNAL);
306
+	cryptd_tfm = cryptd_alloc_ahash("ghash-ce-sync", 0, 0);
280
307
 	if (IS_ERR(cryptd_tfm))
281
308
 		return PTR_ERR(cryptd_tfm);
282
309
 	ctx->cryptd_tfm = cryptd_tfm;
..
..
@@ -324,10 +351,10 @@
324
351
 	if (!(elf_hwcap & HWCAP_NEON))
325
352
 		return -ENODEV;
326
353
 
327
-	if (elf_hwcap2 & HWCAP2_PMULL)
328
-		pmull_ghash_update = pmull_ghash_update_p64;
329
-	else
330
-		pmull_ghash_update = pmull_ghash_update_p8;
354
+	if (elf_hwcap2 & HWCAP2_PMULL) {
355
+		ghash_alg.base.cra_ctxsize += 3 * sizeof(u64[2]);
356
+		static_branch_enable(&use_p64);
357
+	}
331
358
 
332
359
 	err = crypto_register_shash(&ghash_alg);
333
360
 	if (err)