.. | .. |
---|
| 1 | +// SPDX-License-Identifier: GPL-2.0-only |
---|
1 | 2 | /* |
---|
2 | 3 | * Using hardware provided CRC32 instruction to accelerate the CRC32 disposal. |
---|
3 | 4 | * CRC32C polynomial:0x1EDC6F41(BE)/0x82F63B78(LE) |
---|
.. | .. |
---|
9 | 10 | * Copyright (C) 2008 Intel Corporation |
---|
10 | 11 | * Authors: Austin Zhang <austin_zhang@linux.intel.com> |
---|
11 | 12 | * Kent Liu <kent.liu@intel.com> |
---|
12 | | - * |
---|
13 | | - * This program is free software; you can redistribute it and/or modify it |
---|
14 | | - * under the terms and conditions of the GNU General Public License, |
---|
15 | | - * version 2, as published by the Free Software Foundation. |
---|
16 | | - * |
---|
17 | | - * This program is distributed in the hope it will be useful, but WITHOUT |
---|
18 | | - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
---|
19 | | - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for |
---|
20 | | - * more details. |
---|
21 | | - * |
---|
22 | | - * You should have received a copy of the GNU General Public License along with |
---|
23 | | - * this program; if not, write to the Free Software Foundation, Inc., |
---|
24 | | - * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. |
---|
25 | | - * |
---|
26 | 13 | */ |
---|
27 | 14 | #include <linux/init.h> |
---|
28 | 15 | #include <linux/module.h> |
---|
29 | 16 | #include <linux/string.h> |
---|
30 | 17 | #include <linux/kernel.h> |
---|
31 | 18 | #include <crypto/internal/hash.h> |
---|
| 19 | +#include <crypto/internal/simd.h> |
---|
32 | 20 | |
---|
33 | 21 | #include <asm/cpufeatures.h> |
---|
34 | 22 | #include <asm/cpu_device_id.h> |
---|
35 | | -#include <asm/fpu/internal.h> |
---|
| 23 | +#include <asm/simd.h> |
---|
36 | 24 | |
---|
37 | 25 | #define CHKSUM_BLOCK_SIZE 1 |
---|
38 | 26 | #define CHKSUM_DIGEST_SIZE 4 |
---|
.. | .. |
---|
40 | 28 | #define SCALE_F sizeof(unsigned long) |
---|
41 | 29 | |
---|
42 | 30 | #ifdef CONFIG_X86_64 |
---|
43 | | -#define REX_PRE "0x48, " |
---|
| 31 | +#define CRC32_INST "crc32q %1, %q0" |
---|
44 | 32 | #else |
---|
45 | | -#define REX_PRE |
---|
| 33 | +#define CRC32_INST "crc32l %1, %0" |
---|
46 | 34 | #endif |
---|
47 | 35 | |
---|
48 | 36 | #ifdef CONFIG_X86_64 |
---|
.. | .. |
---|
60 | 48 | static u32 crc32c_intel_le_hw_byte(u32 crc, unsigned char const *data, size_t length) |
---|
61 | 49 | { |
---|
62 | 50 | while (length--) { |
---|
63 | | - __asm__ __volatile__( |
---|
64 | | - ".byte 0xf2, 0xf, 0x38, 0xf0, 0xf1" |
---|
65 | | - :"=S"(crc) |
---|
66 | | - :"0"(crc), "c"(*data) |
---|
67 | | - ); |
---|
| 51 | + asm("crc32b %1, %0" |
---|
| 52 | + : "+r" (crc) : "rm" (*data)); |
---|
68 | 53 | data++; |
---|
69 | 54 | } |
---|
70 | 55 | |
---|
.. | .. |
---|
78 | 63 | unsigned long *ptmp = (unsigned long *)p; |
---|
79 | 64 | |
---|
80 | 65 | while (iquotient--) { |
---|
81 | | - __asm__ __volatile__( |
---|
82 | | - ".byte 0xf2, " REX_PRE "0xf, 0x38, 0xf1, 0xf1;" |
---|
83 | | - :"=S"(crc) |
---|
84 | | - :"0"(crc), "c"(*ptmp) |
---|
85 | | - ); |
---|
| 66 | + asm(CRC32_INST |
---|
| 67 | + : "+r" (crc) : "rm" (*ptmp)); |
---|
86 | 68 | ptmp++; |
---|
87 | 69 | } |
---|
88 | 70 | |
---|
.. | .. |
---|
103 | 85 | { |
---|
104 | 86 | u32 *mctx = crypto_shash_ctx(hash); |
---|
105 | 87 | |
---|
106 | | - if (keylen != sizeof(u32)) { |
---|
107 | | - crypto_shash_set_flags(hash, CRYPTO_TFM_RES_BAD_KEY_LEN); |
---|
| 88 | + if (keylen != sizeof(u32)) |
---|
108 | 89 | return -EINVAL; |
---|
109 | | - } |
---|
110 | 90 | *mctx = le32_to_cpup((__le32 *)key); |
---|
111 | 91 | return 0; |
---|
112 | 92 | } |
---|
.. | .. |
---|
177 | 157 | * use faster PCL version if datasize is large enough to |
---|
178 | 158 | * overcome kernel fpu state save/restore overhead |
---|
179 | 159 | */ |
---|
180 | | - if (len >= CRC32C_PCL_BREAKEVEN && irq_fpu_usable()) { |
---|
| 160 | + if (len >= CRC32C_PCL_BREAKEVEN && crypto_simd_usable()) { |
---|
181 | 161 | kernel_fpu_begin(); |
---|
182 | 162 | *crcp = crc_pcl(data, len, *crcp); |
---|
183 | 163 | kernel_fpu_end(); |
---|
.. | .. |
---|
189 | 169 | static int __crc32c_pcl_intel_finup(u32 *crcp, const u8 *data, unsigned int len, |
---|
190 | 170 | u8 *out) |
---|
191 | 171 | { |
---|
192 | | - if (len >= CRC32C_PCL_BREAKEVEN && irq_fpu_usable()) { |
---|
| 172 | + if (len >= CRC32C_PCL_BREAKEVEN && crypto_simd_usable()) { |
---|
193 | 173 | kernel_fpu_begin(); |
---|
194 | 174 | *(__le32 *)out = ~cpu_to_le32(crc_pcl(data, len, *crcp)); |
---|
195 | 175 | kernel_fpu_end(); |
---|
.. | .. |
---|
235 | 215 | }; |
---|
236 | 216 | |
---|
237 | 217 | static const struct x86_cpu_id crc32c_cpu_id[] = { |
---|
238 | | - X86_FEATURE_MATCH(X86_FEATURE_XMM4_2), |
---|
| 218 | + X86_MATCH_FEATURE(X86_FEATURE_XMM4_2, NULL), |
---|
239 | 219 | {} |
---|
240 | 220 | }; |
---|
241 | 221 | MODULE_DEVICE_TABLE(x86cpu, crc32c_cpu_id); |
---|