.. | .. |
---|
13 | 13 | #include <asm/simd.h> |
---|
14 | 14 | #include <asm/unaligned.h> |
---|
15 | 15 | #include <crypto/internal/hash.h> |
---|
| 16 | +#include <crypto/internal/simd.h> |
---|
16 | 17 | #include <crypto/sha.h> |
---|
17 | 18 | #include <crypto/sha512_base.h> |
---|
18 | 19 | #include <linux/cpufeature.h> |
---|
.. | .. |
---|
25 | 26 | MODULE_ALIAS_CRYPTO("sha384"); |
---|
26 | 27 | MODULE_ALIAS_CRYPTO("sha512"); |
---|
27 | 28 | |
---|
28 | | -asmlinkage void sha512_ce_transform(struct sha512_state *sst, u8 const *src, |
---|
29 | | - int blocks); |
---|
| 29 | +asmlinkage int sha512_ce_transform(struct sha512_state *sst, u8 const *src, |
---|
| 30 | + int blocks); |
---|
30 | 31 | |
---|
31 | 32 | asmlinkage void sha512_block_data_order(u64 *digest, u8 const *src, int blocks); |
---|
| 33 | + |
---|
| 34 | +static void __sha512_ce_transform(struct sha512_state *sst, u8 const *src, |
---|
| 35 | + int blocks) |
---|
| 36 | +{ |
---|
| 37 | + while (blocks) { |
---|
| 38 | + int rem; |
---|
| 39 | + |
---|
| 40 | + kernel_neon_begin(); |
---|
| 41 | + rem = sha512_ce_transform(sst, src, blocks); |
---|
| 42 | + kernel_neon_end(); |
---|
| 43 | + src += (blocks - rem) * SHA512_BLOCK_SIZE; |
---|
| 44 | + blocks = rem; |
---|
| 45 | + } |
---|
| 46 | +} |
---|
| 47 | + |
---|
| 48 | +static void __sha512_block_data_order(struct sha512_state *sst, u8 const *src, |
---|
| 49 | + int blocks) |
---|
| 50 | +{ |
---|
| 51 | + sha512_block_data_order(sst->state, src, blocks); |
---|
| 52 | +} |
---|
32 | 53 | |
---|
33 | 54 | static int sha512_ce_update(struct shash_desc *desc, const u8 *data, |
---|
34 | 55 | unsigned int len) |
---|
35 | 56 | { |
---|
36 | | - if (!may_use_simd()) |
---|
37 | | - return sha512_base_do_update(desc, data, len, |
---|
38 | | - (sha512_block_fn *)sha512_block_data_order); |
---|
| 57 | + sha512_block_fn *fn = crypto_simd_usable() ? __sha512_ce_transform |
---|
| 58 | + : __sha512_block_data_order; |
---|
39 | 59 | |
---|
40 | | - kernel_neon_begin(); |
---|
41 | | - sha512_base_do_update(desc, data, len, |
---|
42 | | - (sha512_block_fn *)sha512_ce_transform); |
---|
43 | | - kernel_neon_end(); |
---|
44 | | - |
---|
| 60 | + sha512_base_do_update(desc, data, len, fn); |
---|
45 | 61 | return 0; |
---|
46 | 62 | } |
---|
47 | 63 | |
---|
48 | 64 | static int sha512_ce_finup(struct shash_desc *desc, const u8 *data, |
---|
49 | 65 | unsigned int len, u8 *out) |
---|
50 | 66 | { |
---|
51 | | - if (!may_use_simd()) { |
---|
52 | | - if (len) |
---|
53 | | - sha512_base_do_update(desc, data, len, |
---|
54 | | - (sha512_block_fn *)sha512_block_data_order); |
---|
55 | | - sha512_base_do_finalize(desc, |
---|
56 | | - (sha512_block_fn *)sha512_block_data_order); |
---|
57 | | - return sha512_base_finish(desc, out); |
---|
58 | | - } |
---|
| 67 | + sha512_block_fn *fn = crypto_simd_usable() ? __sha512_ce_transform |
---|
| 68 | + : __sha512_block_data_order; |
---|
59 | 69 | |
---|
60 | | - kernel_neon_begin(); |
---|
61 | | - sha512_base_do_update(desc, data, len, |
---|
62 | | - (sha512_block_fn *)sha512_ce_transform); |
---|
63 | | - sha512_base_do_finalize(desc, (sha512_block_fn *)sha512_ce_transform); |
---|
64 | | - kernel_neon_end(); |
---|
| 70 | + sha512_base_do_update(desc, data, len, fn); |
---|
| 71 | + sha512_base_do_finalize(desc, fn); |
---|
65 | 72 | return sha512_base_finish(desc, out); |
---|
66 | 73 | } |
---|
67 | 74 | |
---|
68 | 75 | static int sha512_ce_final(struct shash_desc *desc, u8 *out) |
---|
69 | 76 | { |
---|
70 | | - if (!may_use_simd()) { |
---|
71 | | - sha512_base_do_finalize(desc, |
---|
72 | | - (sha512_block_fn *)sha512_block_data_order); |
---|
73 | | - return sha512_base_finish(desc, out); |
---|
74 | | - } |
---|
| 77 | + sha512_block_fn *fn = crypto_simd_usable() ? __sha512_ce_transform |
---|
| 78 | + : __sha512_block_data_order; |
---|
75 | 79 | |
---|
76 | | - kernel_neon_begin(); |
---|
77 | | - sha512_base_do_finalize(desc, (sha512_block_fn *)sha512_ce_transform); |
---|
78 | | - kernel_neon_end(); |
---|
| 80 | + sha512_base_do_finalize(desc, fn); |
---|
79 | 81 | return sha512_base_finish(desc, out); |
---|
80 | 82 | } |
---|
81 | 83 | |
---|