.. | .. |
---|
| 1 | +// SPDX-License-Identifier: GPL-2.0-only |
---|
1 | 2 | /* |
---|
2 | 3 | * sha1-ce-glue.c - SHA-1 secure hash using ARMv8 Crypto Extensions |
---|
3 | 4 | * |
---|
4 | 5 | * Copyright (C) 2014 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org> |
---|
5 | | - * |
---|
6 | | - * This program is free software; you can redistribute it and/or modify |
---|
7 | | - * it under the terms of the GNU General Public License version 2 as |
---|
8 | | - * published by the Free Software Foundation. |
---|
9 | 6 | */ |
---|
10 | 7 | |
---|
11 | 8 | #include <asm/neon.h> |
---|
12 | 9 | #include <asm/simd.h> |
---|
13 | 10 | #include <asm/unaligned.h> |
---|
14 | 11 | #include <crypto/internal/hash.h> |
---|
| 12 | +#include <crypto/internal/simd.h> |
---|
15 | 13 | #include <crypto/sha.h> |
---|
16 | 14 | #include <crypto/sha1_base.h> |
---|
17 | 15 | #include <linux/cpufeature.h> |
---|
.. | .. |
---|
28 | 26 | u32 finalize; |
---|
29 | 27 | }; |
---|
30 | 28 | |
---|
31 | | -asmlinkage void sha1_ce_transform(struct sha1_ce_state *sst, u8 const *src, |
---|
32 | | - int blocks); |
---|
33 | | -#ifdef CONFIG_CFI_CLANG |
---|
34 | | -static inline void __cfi_sha1_ce_transform(struct sha1_state *sst, |
---|
35 | | - u8 const *src, int blocks) |
---|
| 29 | +extern const u32 sha1_ce_offsetof_count; |
---|
| 30 | +extern const u32 sha1_ce_offsetof_finalize; |
---|
| 31 | + |
---|
| 32 | +asmlinkage int sha1_ce_transform(struct sha1_ce_state *sst, u8 const *src, |
---|
| 33 | + int blocks); |
---|
| 34 | + |
---|
| 35 | +static void __sha1_ce_transform(struct sha1_state *sst, u8 const *src, |
---|
| 36 | + int blocks) |
---|
36 | 37 | { |
---|
37 | | - sha1_ce_transform((struct sha1_ce_state *)sst, src, blocks); |
---|
| 38 | + while (blocks) { |
---|
| 39 | + int rem; |
---|
| 40 | + |
---|
| 41 | + kernel_neon_begin(); |
---|
| 42 | + rem = sha1_ce_transform(container_of(sst, struct sha1_ce_state, |
---|
| 43 | + sst), src, blocks); |
---|
| 44 | + kernel_neon_end(); |
---|
| 45 | + src += (blocks - rem) * SHA1_BLOCK_SIZE; |
---|
| 46 | + blocks = rem; |
---|
| 47 | + } |
---|
38 | 48 | } |
---|
39 | | -#define sha1_ce_transform __cfi_sha1_ce_transform |
---|
40 | | -#endif |
---|
41 | 49 | |
---|
42 | 50 | const u32 sha1_ce_offsetof_count = offsetof(struct sha1_ce_state, sst.count); |
---|
43 | 51 | const u32 sha1_ce_offsetof_finalize = offsetof(struct sha1_ce_state, finalize); |
---|
.. | .. |
---|
47 | 55 | { |
---|
48 | 56 | struct sha1_ce_state *sctx = shash_desc_ctx(desc); |
---|
49 | 57 | |
---|
50 | | - if (!may_use_simd()) |
---|
| 58 | + if (!crypto_simd_usable()) |
---|
51 | 59 | return crypto_sha1_update(desc, data, len); |
---|
52 | 60 | |
---|
53 | 61 | sctx->finalize = 0; |
---|
54 | | - kernel_neon_begin(); |
---|
55 | | - sha1_base_do_update(desc, data, len, |
---|
56 | | - (sha1_block_fn *)sha1_ce_transform); |
---|
57 | | - kernel_neon_end(); |
---|
| 62 | + sha1_base_do_update(desc, data, len, __sha1_ce_transform); |
---|
58 | 63 | |
---|
59 | 64 | return 0; |
---|
60 | 65 | } |
---|
.. | .. |
---|
65 | 70 | struct sha1_ce_state *sctx = shash_desc_ctx(desc); |
---|
66 | 71 | bool finalize = !sctx->sst.count && !(len % SHA1_BLOCK_SIZE) && len; |
---|
67 | 72 | |
---|
68 | | - if (!may_use_simd()) |
---|
| 73 | + if (!crypto_simd_usable()) |
---|
69 | 74 | return crypto_sha1_finup(desc, data, len, out); |
---|
70 | 75 | |
---|
71 | 76 | /* |
---|
.. | .. |
---|
74 | 79 | */ |
---|
75 | 80 | sctx->finalize = finalize; |
---|
76 | 81 | |
---|
77 | | - kernel_neon_begin(); |
---|
78 | | - sha1_base_do_update(desc, data, len, |
---|
79 | | - (sha1_block_fn *)sha1_ce_transform); |
---|
| 82 | + sha1_base_do_update(desc, data, len, __sha1_ce_transform); |
---|
80 | 83 | if (!finalize) |
---|
81 | | - sha1_base_do_finalize(desc, (sha1_block_fn *)sha1_ce_transform); |
---|
82 | | - kernel_neon_end(); |
---|
| 84 | + sha1_base_do_finalize(desc, __sha1_ce_transform); |
---|
83 | 85 | return sha1_base_finish(desc, out); |
---|
84 | 86 | } |
---|
85 | 87 | |
---|
.. | .. |
---|
87 | 89 | { |
---|
88 | 90 | struct sha1_ce_state *sctx = shash_desc_ctx(desc); |
---|
89 | 91 | |
---|
90 | | - if (!may_use_simd()) |
---|
| 92 | + if (!crypto_simd_usable()) |
---|
91 | 93 | return crypto_sha1_finup(desc, NULL, 0, out); |
---|
92 | 94 | |
---|
93 | 95 | sctx->finalize = 0; |
---|
94 | | - kernel_neon_begin(); |
---|
95 | | - sha1_base_do_finalize(desc, (sha1_block_fn *)sha1_ce_transform); |
---|
96 | | - kernel_neon_end(); |
---|
| 96 | + sha1_base_do_finalize(desc, __sha1_ce_transform); |
---|
97 | 97 | return sha1_base_finish(desc, out); |
---|
| 98 | +} |
---|
| 99 | + |
---|
| 100 | +static int sha1_ce_export(struct shash_desc *desc, void *out) |
---|
| 101 | +{ |
---|
| 102 | + struct sha1_ce_state *sctx = shash_desc_ctx(desc); |
---|
| 103 | + |
---|
| 104 | + memcpy(out, &sctx->sst, sizeof(struct sha1_state)); |
---|
| 105 | + return 0; |
---|
| 106 | +} |
---|
| 107 | + |
---|
| 108 | +static int sha1_ce_import(struct shash_desc *desc, const void *in) |
---|
| 109 | +{ |
---|
| 110 | + struct sha1_ce_state *sctx = shash_desc_ctx(desc); |
---|
| 111 | + |
---|
| 112 | + memcpy(&sctx->sst, in, sizeof(struct sha1_state)); |
---|
| 113 | + sctx->finalize = 0; |
---|
| 114 | + return 0; |
---|
98 | 115 | } |
---|
99 | 116 | |
---|
100 | 117 | static struct shash_alg alg = { |
---|
.. | .. |
---|
102 | 119 | .update = sha1_ce_update, |
---|
103 | 120 | .final = sha1_ce_final, |
---|
104 | 121 | .finup = sha1_ce_finup, |
---|
| 122 | + .import = sha1_ce_import, |
---|
| 123 | + .export = sha1_ce_export, |
---|
105 | 124 | .descsize = sizeof(struct sha1_ce_state), |
---|
| 125 | + .statesize = sizeof(struct sha1_state), |
---|
106 | 126 | .digestsize = SHA1_DIGEST_SIZE, |
---|
107 | 127 | .base = { |
---|
108 | 128 | .cra_name = "sha1", |
---|