.. | .. |
---|
| 1 | +// SPDX-License-Identifier: GPL-2.0-or-later |
---|
1 | 2 | /* |
---|
2 | 3 | * Linux/arm64 port of the OpenSSL SHA256 implementation for AArch64 |
---|
3 | 4 | * |
---|
4 | 5 | * Copyright (c) 2016 Linaro Ltd. <ard.biesheuvel@linaro.org> |
---|
5 | | - * |
---|
6 | | - * This program is free software; you can redistribute it and/or modify it |
---|
7 | | - * under the terms of the GNU General Public License as published by the Free |
---|
8 | | - * Software Foundation; either version 2 of the License, or (at your option) |
---|
9 | | - * any later version. |
---|
10 | | - * |
---|
11 | 6 | */ |
---|
12 | 7 | |
---|
13 | 8 | #include <asm/hwcap.h> |
---|
14 | 9 | #include <asm/neon.h> |
---|
15 | 10 | #include <asm/simd.h> |
---|
16 | 11 | #include <crypto/internal/hash.h> |
---|
| 12 | +#include <crypto/internal/simd.h> |
---|
17 | 13 | #include <crypto/sha.h> |
---|
18 | 14 | #include <crypto/sha256_base.h> |
---|
19 | | -#include <linux/cryptohash.h> |
---|
20 | 15 | #include <linux/types.h> |
---|
21 | 16 | #include <linux/string.h> |
---|
22 | 17 | |
---|
.. | .. |
---|
31 | 26 | unsigned int num_blks); |
---|
32 | 27 | EXPORT_SYMBOL(sha256_block_data_order); |
---|
33 | 28 | |
---|
| 29 | +static void __sha256_block_data_order(struct sha256_state *sst, u8 const *src, |
---|
| 30 | + int blocks) |
---|
| 31 | +{ |
---|
| 32 | + sha256_block_data_order(sst->state, src, blocks); |
---|
| 33 | +} |
---|
| 34 | + |
---|
34 | 35 | asmlinkage void sha256_block_neon(u32 *digest, const void *data, |
---|
35 | 36 | unsigned int num_blks); |
---|
36 | 37 | |
---|
37 | | -static int sha256_update(struct shash_desc *desc, const u8 *data, |
---|
38 | | - unsigned int len) |
---|
| 38 | +static void __sha256_block_neon(struct sha256_state *sst, u8 const *src, |
---|
| 39 | + int blocks) |
---|
39 | 40 | { |
---|
40 | | - return sha256_base_do_update(desc, data, len, |
---|
41 | | - (sha256_block_fn *)sha256_block_data_order); |
---|
| 41 | + sha256_block_neon(sst->state, src, blocks); |
---|
42 | 42 | } |
---|
43 | 43 | |
---|
44 | | -static int sha256_finup(struct shash_desc *desc, const u8 *data, |
---|
45 | | - unsigned int len, u8 *out) |
---|
| 44 | +static int crypto_sha256_arm64_update(struct shash_desc *desc, const u8 *data, |
---|
| 45 | + unsigned int len) |
---|
| 46 | +{ |
---|
| 47 | + return sha256_base_do_update(desc, data, len, |
---|
| 48 | + __sha256_block_data_order); |
---|
| 49 | +} |
---|
| 50 | + |
---|
| 51 | +static int crypto_sha256_arm64_finup(struct shash_desc *desc, const u8 *data, |
---|
| 52 | + unsigned int len, u8 *out) |
---|
46 | 53 | { |
---|
47 | 54 | if (len) |
---|
48 | 55 | sha256_base_do_update(desc, data, len, |
---|
49 | | - (sha256_block_fn *)sha256_block_data_order); |
---|
50 | | - sha256_base_do_finalize(desc, |
---|
51 | | - (sha256_block_fn *)sha256_block_data_order); |
---|
| 56 | + __sha256_block_data_order); |
---|
| 57 | + sha256_base_do_finalize(desc, __sha256_block_data_order); |
---|
52 | 58 | |
---|
53 | 59 | return sha256_base_finish(desc, out); |
---|
54 | 60 | } |
---|
55 | 61 | |
---|
56 | | -static int sha256_final(struct shash_desc *desc, u8 *out) |
---|
| 62 | +static int crypto_sha256_arm64_final(struct shash_desc *desc, u8 *out) |
---|
57 | 63 | { |
---|
58 | | - return sha256_finup(desc, NULL, 0, out); |
---|
| 64 | + return crypto_sha256_arm64_finup(desc, NULL, 0, out); |
---|
59 | 65 | } |
---|
60 | 66 | |
---|
61 | 67 | static struct shash_alg algs[] = { { |
---|
62 | 68 | .digestsize = SHA256_DIGEST_SIZE, |
---|
63 | 69 | .init = sha256_base_init, |
---|
64 | | - .update = sha256_update, |
---|
65 | | - .final = sha256_final, |
---|
66 | | - .finup = sha256_finup, |
---|
| 70 | + .update = crypto_sha256_arm64_update, |
---|
| 71 | + .final = crypto_sha256_arm64_final, |
---|
| 72 | + .finup = crypto_sha256_arm64_finup, |
---|
67 | 73 | .descsize = sizeof(struct sha256_state), |
---|
68 | 74 | .base.cra_name = "sha256", |
---|
69 | 75 | .base.cra_driver_name = "sha256-arm64", |
---|
.. | .. |
---|
73 | 79 | }, { |
---|
74 | 80 | .digestsize = SHA224_DIGEST_SIZE, |
---|
75 | 81 | .init = sha224_base_init, |
---|
76 | | - .update = sha256_update, |
---|
77 | | - .final = sha256_final, |
---|
78 | | - .finup = sha256_finup, |
---|
| 82 | + .update = crypto_sha256_arm64_update, |
---|
| 83 | + .final = crypto_sha256_arm64_final, |
---|
| 84 | + .finup = crypto_sha256_arm64_finup, |
---|
79 | 85 | .descsize = sizeof(struct sha256_state), |
---|
80 | 86 | .base.cra_name = "sha224", |
---|
81 | 87 | .base.cra_driver_name = "sha224-arm64", |
---|
.. | .. |
---|
89 | 95 | { |
---|
90 | 96 | struct sha256_state *sctx = shash_desc_ctx(desc); |
---|
91 | 97 | |
---|
92 | | - if (!may_use_simd()) |
---|
| 98 | + if (!crypto_simd_usable()) |
---|
93 | 99 | return sha256_base_do_update(desc, data, len, |
---|
94 | | - (sha256_block_fn *)sha256_block_data_order); |
---|
| 100 | + __sha256_block_data_order); |
---|
95 | 101 | |
---|
96 | 102 | while (len > 0) { |
---|
97 | 103 | unsigned int chunk = len; |
---|
.. | .. |
---|
101 | 107 | * input when running on a preemptible kernel, but process the |
---|
102 | 108 | * data block by block instead. |
---|
103 | 109 | */ |
---|
104 | | - if (IS_ENABLED(CONFIG_PREEMPT) && |
---|
| 110 | + if (IS_ENABLED(CONFIG_PREEMPTION) && |
---|
105 | 111 | chunk + sctx->count % SHA256_BLOCK_SIZE > SHA256_BLOCK_SIZE) |
---|
106 | 112 | chunk = SHA256_BLOCK_SIZE - |
---|
107 | 113 | sctx->count % SHA256_BLOCK_SIZE; |
---|
108 | 114 | |
---|
109 | 115 | kernel_neon_begin(); |
---|
110 | | - sha256_base_do_update(desc, data, chunk, |
---|
111 | | - (sha256_block_fn *)sha256_block_neon); |
---|
| 116 | + sha256_base_do_update(desc, data, chunk, __sha256_block_neon); |
---|
112 | 117 | kernel_neon_end(); |
---|
113 | 118 | data += chunk; |
---|
114 | 119 | len -= chunk; |
---|
.. | .. |
---|
119 | 124 | static int sha256_finup_neon(struct shash_desc *desc, const u8 *data, |
---|
120 | 125 | unsigned int len, u8 *out) |
---|
121 | 126 | { |
---|
122 | | - if (!may_use_simd()) { |
---|
| 127 | + if (!crypto_simd_usable()) { |
---|
123 | 128 | if (len) |
---|
124 | 129 | sha256_base_do_update(desc, data, len, |
---|
125 | | - (sha256_block_fn *)sha256_block_data_order); |
---|
126 | | - sha256_base_do_finalize(desc, |
---|
127 | | - (sha256_block_fn *)sha256_block_data_order); |
---|
| 130 | + __sha256_block_data_order); |
---|
| 131 | + sha256_base_do_finalize(desc, __sha256_block_data_order); |
---|
128 | 132 | } else { |
---|
129 | 133 | if (len) |
---|
130 | 134 | sha256_update_neon(desc, data, len); |
---|
131 | 135 | kernel_neon_begin(); |
---|
132 | | - sha256_base_do_finalize(desc, |
---|
133 | | - (sha256_block_fn *)sha256_block_neon); |
---|
| 136 | + sha256_base_do_finalize(desc, __sha256_block_neon); |
---|
134 | 137 | kernel_neon_end(); |
---|
135 | 138 | } |
---|
136 | 139 | return sha256_base_finish(desc, out); |
---|
.. | .. |
---|
173 | 176 | if (ret) |
---|
174 | 177 | return ret; |
---|
175 | 178 | |
---|
176 | | - if (elf_hwcap & HWCAP_ASIMD) { |
---|
| 179 | + if (cpu_have_named_feature(ASIMD)) { |
---|
177 | 180 | ret = crypto_register_shashes(neon_algs, ARRAY_SIZE(neon_algs)); |
---|
178 | 181 | if (ret) |
---|
179 | 182 | crypto_unregister_shashes(algs, ARRAY_SIZE(algs)); |
---|
.. | .. |
---|
183 | 186 | |
---|
184 | 187 | static void __exit sha256_mod_fini(void) |
---|
185 | 188 | { |
---|
186 | | - if (elf_hwcap & HWCAP_ASIMD) |
---|
| 189 | + if (cpu_have_named_feature(ASIMD)) |
---|
187 | 190 | crypto_unregister_shashes(neon_algs, ARRAY_SIZE(neon_algs)); |
---|
188 | 191 | crypto_unregister_shashes(algs, ARRAY_SIZE(algs)); |
---|
189 | 192 | } |
---|