.. | .. |
---|
| 1 | +// SPDX-License-Identifier: GPL-2.0-or-later |
---|
1 | 2 | /* |
---|
2 | 3 | * x64 SIMD accelerated ChaCha and XChaCha stream ciphers, |
---|
3 | 4 | * including ChaCha20 (RFC7539) |
---|
4 | 5 | * |
---|
5 | 6 | * Copyright (C) 2015 Martin Willi |
---|
6 | | - * |
---|
7 | | - * This program is free software; you can redistribute it and/or modify |
---|
8 | | - * it under the terms of the GNU General Public License as published by |
---|
9 | | - * the Free Software Foundation; either version 2 of the License, or |
---|
10 | | - * (at your option) any later version. |
---|
11 | 7 | */ |
---|
12 | 8 | |
---|
13 | 9 | #include <crypto/algapi.h> |
---|
14 | 10 | #include <crypto/internal/chacha.h> |
---|
| 11 | +#include <crypto/internal/simd.h> |
---|
15 | 12 | #include <crypto/internal/skcipher.h> |
---|
16 | 13 | #include <linux/kernel.h> |
---|
17 | 14 | #include <linux/module.h> |
---|
18 | | -#include <asm/fpu/api.h> |
---|
| 15 | +#include <linux/sizes.h> |
---|
19 | 16 | #include <asm/simd.h> |
---|
20 | 17 | |
---|
21 | 18 | asmlinkage void chacha_block_xor_ssse3(u32 *state, u8 *dst, const u8 *src, |
---|
.. | .. |
---|
81 | 78 | } |
---|
82 | 79 | } |
---|
83 | 80 | |
---|
84 | | - if (IS_ENABLED(CONFIG_AS_AVX2) && |
---|
85 | | - static_branch_likely(&chacha_use_avx2)) { |
---|
| 81 | + if (static_branch_likely(&chacha_use_avx2)) { |
---|
86 | 82 | while (bytes >= CHACHA_BLOCK_SIZE * 8) { |
---|
87 | 83 | chacha_8block_xor_avx2(state, dst, src, bytes, nrounds); |
---|
88 | 84 | bytes -= CHACHA_BLOCK_SIZE * 8; |
---|
.. | .. |
---|
127 | 123 | |
---|
128 | 124 | void hchacha_block_arch(const u32 *state, u32 *stream, int nrounds) |
---|
129 | 125 | { |
---|
130 | | - if (!static_branch_likely(&chacha_use_simd) || !may_use_simd()) { |
---|
| 126 | + if (!static_branch_likely(&chacha_use_simd) || !crypto_simd_usable()) { |
---|
131 | 127 | hchacha_block_generic(state, stream, nrounds); |
---|
132 | 128 | } else { |
---|
133 | 129 | kernel_fpu_begin(); |
---|
.. | .. |
---|
146 | 142 | void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, unsigned int bytes, |
---|
147 | 143 | int nrounds) |
---|
148 | 144 | { |
---|
149 | | - if (!static_branch_likely(&chacha_use_simd) || !may_use_simd() || |
---|
| 145 | + if (!static_branch_likely(&chacha_use_simd) || !crypto_simd_usable() || |
---|
150 | 146 | bytes <= CHACHA_BLOCK_SIZE) |
---|
151 | 147 | return chacha_crypt_generic(state, dst, src, bytes, nrounds); |
---|
152 | 148 | |
---|
.. | .. |
---|
182 | 178 | nbytes = round_down(nbytes, walk.stride); |
---|
183 | 179 | |
---|
184 | 180 | if (!static_branch_likely(&chacha_use_simd) || |
---|
185 | | - !may_use_simd()) { |
---|
| 181 | + !crypto_simd_usable()) { |
---|
186 | 182 | chacha_crypt_generic(state, walk.dst.virt.addr, |
---|
187 | 183 | walk.src.virt.addr, nbytes, |
---|
188 | 184 | ctx->nrounds); |
---|
.. | .. |
---|
217 | 213 | |
---|
218 | 214 | chacha_init_generic(state, ctx->key, req->iv); |
---|
219 | 215 | |
---|
220 | | - if (req->cryptlen > CHACHA_BLOCK_SIZE && irq_fpu_usable()) { |
---|
| 216 | + if (req->cryptlen > CHACHA_BLOCK_SIZE && crypto_simd_usable()) { |
---|
221 | 217 | kernel_fpu_begin(); |
---|
222 | 218 | hchacha_block_ssse3(state, subctx.key, ctx->nrounds); |
---|
223 | 219 | kernel_fpu_end(); |
---|
.. | .. |
---|
287 | 283 | |
---|
288 | 284 | static_branch_enable(&chacha_use_simd); |
---|
289 | 285 | |
---|
290 | | - if (IS_ENABLED(CONFIG_AS_AVX2) && |
---|
291 | | - boot_cpu_has(X86_FEATURE_AVX) && |
---|
| 286 | + if (boot_cpu_has(X86_FEATURE_AVX) && |
---|
292 | 287 | boot_cpu_has(X86_FEATURE_AVX2) && |
---|
293 | 288 | cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) { |
---|
294 | 289 | static_branch_enable(&chacha_use_avx2); |
---|
.. | .. |
---|
298 | 293 | boot_cpu_has(X86_FEATURE_AVX512BW)) /* kmovq */ |
---|
299 | 294 | static_branch_enable(&chacha_use_avx512vl); |
---|
300 | 295 | } |
---|
301 | | - return IS_REACHABLE(CONFIG_CRYPTO_BLKCIPHER) ? |
---|
| 296 | + return IS_REACHABLE(CONFIG_CRYPTO_SKCIPHER) ? |
---|
302 | 297 | crypto_register_skciphers(algs, ARRAY_SIZE(algs)) : 0; |
---|
303 | 298 | } |
---|
304 | 299 | |
---|
305 | 300 | static void __exit chacha_simd_mod_fini(void) |
---|
306 | 301 | { |
---|
307 | | - if (IS_REACHABLE(CONFIG_CRYPTO_BLKCIPHER) && boot_cpu_has(X86_FEATURE_SSSE3)) |
---|
| 302 | + if (IS_REACHABLE(CONFIG_CRYPTO_SKCIPHER) && boot_cpu_has(X86_FEATURE_SSSE3)) |
---|
308 | 303 | crypto_unregister_skciphers(algs, ARRAY_SIZE(algs)); |
---|
309 | 304 | } |
---|
310 | 305 | |
---|