enable docker ppp

hc

2023-12-09 95099d4622f8cb224d94e314c7a8e0df60b13f87

 kernel/arch/arm64/crypto/crct10dif-ce-glue.c
..
..
@@ -1,11 +1,8 @@
1
+// SPDX-License-Identifier: GPL-2.0-only
1
2
 /*
2
3
  * Accelerated CRC-T10DIF using arm64 NEON and Crypto Extensions instructions
3
4
  *
4
5
  * Copyright (C) 2016 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
5
- *
6
- * This program is free software; you can redistribute it and/or modify
7
- * it under the terms of the GNU General Public License version 2 as
8
- * published by the Free Software Foundation.
9
6
  */
10
7
 
11
8
 #include <linux/cpufeature.h>
..
..
@@ -16,13 +13,15 @@
16
13
 #include <linux/string.h>
17
14
 
18
15
 #include <crypto/internal/hash.h>
16
+#include <crypto/internal/simd.h>
19
17
 
20
18
 #include <asm/neon.h>
21
19
 #include <asm/simd.h>
22
20
 
23
21
 #define CRC_T10DIF_PMULL_CHUNK_SIZE	16U
24
22
 
25
-asmlinkage u16 crc_t10dif_pmull(u16 init_crc, const u8 buf[], u64 len);
23
+asmlinkage u16 crc_t10dif_pmull_p8(u16 init_crc, const u8 *buf, size_t len);
24
+asmlinkage u16 crc_t10dif_pmull_p64(u16 init_crc, const u8 *buf, size_t len);
26
25
 
27
26
 static int crct10dif_init(struct shash_desc *desc)
28
27
 {
..
..
@@ -32,15 +31,49 @@
32
31
 	return 0;
33
32
 }
34
33
 
35
-static int crct10dif_update(struct shash_desc *desc, const u8 *data,
34
+static int crct10dif_update_pmull_p8(struct shash_desc *desc, const u8 *data,
36
35
 			    unsigned int length)
37
36
 {
38
37
 	u16 *crc = shash_desc_ctx(desc);
39
38
 
40
-	if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE && may_use_simd()) {
41
-		kernel_neon_begin();
42
-		*crc = crc_t10dif_pmull(*crc, data, length);
43
-		kernel_neon_end();
39
+	if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE && crypto_simd_usable()) {
40
+		do {
41
+			unsigned int chunk = length;
42
+
43
+			if (chunk > SZ_4K + CRC_T10DIF_PMULL_CHUNK_SIZE)
44
+				chunk = SZ_4K;
45
+
46
+			kernel_neon_begin();
47
+			*crc = crc_t10dif_pmull_p8(*crc, data, chunk);
48
+			kernel_neon_end();
49
+			data += chunk;
50
+			length -= chunk;
51
+		} while (length);
52
+	} else {
53
+		*crc = crc_t10dif_generic(*crc, data, length);
54
+	}
55
+
56
+	return 0;
57
+}
58
+
59
+static int crct10dif_update_pmull_p64(struct shash_desc *desc, const u8 *data,
60
+			    unsigned int length)
61
+{
62
+	u16 *crc = shash_desc_ctx(desc);
63
+
64
+	if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE && crypto_simd_usable()) {
65
+		do {
66
+			unsigned int chunk = length;
67
+
68
+			if (chunk > SZ_4K + CRC_T10DIF_PMULL_CHUNK_SIZE)
69
+				chunk = SZ_4K;
70
+
71
+			kernel_neon_begin();
72
+			*crc = crc_t10dif_pmull_p64(*crc, data, chunk);
73
+			kernel_neon_end();
74
+			data += chunk;
75
+			length -= chunk;
76
+		} while (length);
44
77
 	} else {
45
78
 		*crc = crc_t10dif_generic(*crc, data, length);
46
79
 	}
..
..
@@ -56,10 +89,22 @@
56
89
 	return 0;
57
90
 }
58
91
 
59
-static struct shash_alg crc_t10dif_alg = {
92
+static struct shash_alg crc_t10dif_alg[] = {{
60
93
 	.digestsize		= CRC_T10DIF_DIGEST_SIZE,
61
94
 	.init			= crct10dif_init,
62
-	.update			= crct10dif_update,
95
+	.update			= crct10dif_update_pmull_p8,
96
+	.final			= crct10dif_final,
97
+	.descsize		= CRC_T10DIF_DIGEST_SIZE,
98
+
99
+	.base.cra_name		= "crct10dif",
100
+	.base.cra_driver_name	= "crct10dif-arm64-neon",
101
+	.base.cra_priority	= 100,
102
+	.base.cra_blocksize	= CRC_T10DIF_BLOCK_SIZE,
103
+	.base.cra_module	= THIS_MODULE,
104
+}, {
105
+	.digestsize		= CRC_T10DIF_DIGEST_SIZE,
106
+	.init			= crct10dif_init,
107
+	.update			= crct10dif_update_pmull_p64,
63
108
 	.final			= crct10dif_final,
64
109
 	.descsize		= CRC_T10DIF_DIGEST_SIZE,
65
110
 
..
..
@@ -68,20 +113,31 @@
68
113
 	.base.cra_priority	= 200,
69
114
 	.base.cra_blocksize	= CRC_T10DIF_BLOCK_SIZE,
70
115
 	.base.cra_module	= THIS_MODULE,
71
-};
116
+}};
72
117
 
73
118
 static int __init crc_t10dif_mod_init(void)
74
119
 {
75
-	return crypto_register_shash(&crc_t10dif_alg);
120
+	if (cpu_have_named_feature(PMULL))
121
+		return crypto_register_shashes(crc_t10dif_alg,
122
+					       ARRAY_SIZE(crc_t10dif_alg));
123
+	else
124
+		/* only register the first array element */
125
+		return crypto_register_shash(crc_t10dif_alg);
76
126
 }
77
127
 
78
128
 static void __exit crc_t10dif_mod_exit(void)
79
129
 {
80
-	crypto_unregister_shash(&crc_t10dif_alg);
130
+	if (cpu_have_named_feature(PMULL))
131
+		crypto_unregister_shashes(crc_t10dif_alg,
132
+					  ARRAY_SIZE(crc_t10dif_alg));
133
+	else
134
+		crypto_unregister_shash(crc_t10dif_alg);
81
135
 }
82
136
 
83
-module_cpu_feature_match(PMULL, crc_t10dif_mod_init);
137
+module_cpu_feature_match(ASIMD, crc_t10dif_mod_init);
84
138
 module_exit(crc_t10dif_mod_exit);
85
139
 
86
140
 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
87
141
 MODULE_LICENSE("GPL v2");
142
+MODULE_ALIAS_CRYPTO("crct10dif");
143
+MODULE_ALIAS_CRYPTO("crct10dif-arm64-ce");