.. | .. |
---|
| 1 | +/* SPDX-License-Identifier: GPL-2.0-only */ |
---|
1 | 2 | /* |
---|
2 | 3 | * Accelerated GHASH implementation with Intel PCLMULQDQ-NI |
---|
3 | 4 | * instructions. This file contains accelerated part of ghash |
---|
.. | .. |
---|
10 | 11 | * Vinodh Gopal |
---|
11 | 12 | * Erdinc Ozturk |
---|
12 | 13 | * Deniz Karakoyunlu |
---|
13 | | - * |
---|
14 | | - * This program is free software; you can redistribute it and/or modify it |
---|
15 | | - * under the terms of the GNU General Public License version 2 as published |
---|
16 | | - * by the Free Software Foundation. |
---|
17 | 14 | */ |
---|
18 | 15 | |
---|
19 | 16 | #include <linux/linkage.h> |
---|
20 | | -#include <asm/inst.h> |
---|
21 | 17 | #include <asm/frame.h> |
---|
22 | 18 | |
---|
23 | 19 | .section .rodata.cst16.bswap_mask, "aM", @progbits, 16 |
---|
.. | .. |
---|
47 | 43 | * T2 |
---|
48 | 44 | * T3 |
---|
49 | 45 | */ |
---|
50 | | -__clmul_gf128mul_ble: |
---|
| 46 | +SYM_FUNC_START_LOCAL(__clmul_gf128mul_ble) |
---|
51 | 47 | movaps DATA, T1 |
---|
52 | 48 | pshufd $0b01001110, DATA, T2 |
---|
53 | 49 | pshufd $0b01001110, SHASH, T3 |
---|
54 | 50 | pxor DATA, T2 |
---|
55 | 51 | pxor SHASH, T3 |
---|
56 | 52 | |
---|
57 | | - PCLMULQDQ 0x00 SHASH DATA # DATA = a0 * b0 |
---|
58 | | - PCLMULQDQ 0x11 SHASH T1 # T1 = a1 * b1 |
---|
59 | | - PCLMULQDQ 0x00 T3 T2 # T2 = (a1 + a0) * (b1 + b0) |
---|
| 53 | + pclmulqdq $0x00, SHASH, DATA # DATA = a0 * b0 |
---|
| 54 | + pclmulqdq $0x11, SHASH, T1 # T1 = a1 * b1 |
---|
| 55 | + pclmulqdq $0x00, T3, T2 # T2 = (a1 + a0) * (b1 + b0) |
---|
60 | 56 | pxor DATA, T2 |
---|
61 | 57 | pxor T1, T2 # T2 = a0 * b1 + a1 * b0 |
---|
62 | 58 | |
---|
.. | .. |
---|
89 | 85 | psrlq $1, T2 |
---|
90 | 86 | pxor T2, T1 |
---|
91 | 87 | pxor T1, DATA |
---|
92 | | - ret |
---|
93 | | -ENDPROC(__clmul_gf128mul_ble) |
---|
| 88 | + RET |
---|
| 89 | +SYM_FUNC_END(__clmul_gf128mul_ble) |
---|
94 | 90 | |
---|
95 | 91 | /* void clmul_ghash_mul(char *dst, const u128 *shash) */ |
---|
96 | | -ENTRY(clmul_ghash_mul) |
---|
| 92 | +SYM_FUNC_START(clmul_ghash_mul) |
---|
97 | 93 | FRAME_BEGIN |
---|
98 | 94 | movups (%rdi), DATA |
---|
99 | 95 | movups (%rsi), SHASH |
---|
100 | 96 | movaps .Lbswap_mask, BSWAP |
---|
101 | | - PSHUFB_XMM BSWAP DATA |
---|
| 97 | + pshufb BSWAP, DATA |
---|
102 | 98 | call __clmul_gf128mul_ble |
---|
103 | | - PSHUFB_XMM BSWAP DATA |
---|
| 99 | + pshufb BSWAP, DATA |
---|
104 | 100 | movups DATA, (%rdi) |
---|
105 | 101 | FRAME_END |
---|
106 | | - ret |
---|
107 | | -ENDPROC(clmul_ghash_mul) |
---|
| 102 | + RET |
---|
| 103 | +SYM_FUNC_END(clmul_ghash_mul) |
---|
108 | 104 | |
---|
109 | 105 | /* |
---|
110 | 106 | * void clmul_ghash_update(char *dst, const char *src, unsigned int srclen, |
---|
111 | 107 | * const u128 *shash); |
---|
112 | 108 | */ |
---|
113 | | -ENTRY(clmul_ghash_update) |
---|
| 109 | +SYM_FUNC_START(clmul_ghash_update) |
---|
114 | 110 | FRAME_BEGIN |
---|
115 | 111 | cmp $16, %rdx |
---|
116 | 112 | jb .Lupdate_just_ret # check length |
---|
117 | 113 | movaps .Lbswap_mask, BSWAP |
---|
118 | 114 | movups (%rdi), DATA |
---|
119 | 115 | movups (%rcx), SHASH |
---|
120 | | - PSHUFB_XMM BSWAP DATA |
---|
| 116 | + pshufb BSWAP, DATA |
---|
121 | 117 | .align 4 |
---|
122 | 118 | .Lupdate_loop: |
---|
123 | 119 | movups (%rsi), IN1 |
---|
124 | | - PSHUFB_XMM BSWAP IN1 |
---|
| 120 | + pshufb BSWAP, IN1 |
---|
125 | 121 | pxor IN1, DATA |
---|
126 | 122 | call __clmul_gf128mul_ble |
---|
127 | 123 | sub $16, %rdx |
---|
128 | 124 | add $16, %rsi |
---|
129 | 125 | cmp $16, %rdx |
---|
130 | 126 | jge .Lupdate_loop |
---|
131 | | - PSHUFB_XMM BSWAP DATA |
---|
| 127 | + pshufb BSWAP, DATA |
---|
132 | 128 | movups DATA, (%rdi) |
---|
133 | 129 | .Lupdate_just_ret: |
---|
134 | 130 | FRAME_END |
---|
135 | | - ret |
---|
136 | | -ENDPROC(clmul_ghash_update) |
---|
| 131 | + RET |
---|
| 132 | +SYM_FUNC_END(clmul_ghash_update) |
---|