| .. | .. |
|---|
| 43 | 43 | * SOFTWARE. |
|---|
| 44 | 44 | */ |
|---|
| 45 | 45 | |
|---|
| 46 | | -#include <asm/inst.h> |
|---|
| 47 | 46 | #include <linux/linkage.h> |
|---|
| 48 | 47 | #include <asm/nospec-branch.h> |
|---|
| 49 | 48 | |
|---|
| .. | .. |
|---|
| 74 | 73 | # unsigned int crc_pcl(u8 *buffer, int len, unsigned int crc_init); |
|---|
| 75 | 74 | |
|---|
| 76 | 75 | .text |
|---|
| 77 | | -ENTRY(crc_pcl) |
|---|
| 78 | | -#define bufp %rdi |
|---|
| 76 | +SYM_FUNC_START(crc_pcl) |
|---|
| 77 | +#define bufp rdi |
|---|
| 79 | 78 | #define bufp_dw %edi |
|---|
| 80 | 79 | #define bufp_w %di |
|---|
| 81 | 80 | #define bufp_b %dil |
|---|
| .. | .. |
|---|
| 105 | 104 | ## 1) ALIGN: |
|---|
| 106 | 105 | ################################################################ |
|---|
| 107 | 106 | |
|---|
| 108 | | - mov bufp, bufptmp # rdi = *buf |
|---|
| 109 | | - neg bufp |
|---|
| 110 | | - and $7, bufp # calculate the unalignment amount of |
|---|
| 107 | + mov %bufp, bufptmp # rdi = *buf |
|---|
| 108 | + neg %bufp |
|---|
| 109 | + and $7, %bufp # calculate the unalignment amount of |
|---|
| 111 | 110 | # the address |
|---|
| 112 | 111 | je proc_block # Skip if aligned |
|---|
| 113 | 112 | |
|---|
| .. | .. |
|---|
| 123 | 122 | do_align: |
|---|
| 124 | 123 | #### Calculate CRC of unaligned bytes of the buffer (if any) |
|---|
| 125 | 124 | movq (bufptmp), tmp # load a quadward from the buffer |
|---|
| 126 | | - add bufp, bufptmp # align buffer pointer for quadword |
|---|
| 125 | + add %bufp, bufptmp # align buffer pointer for quadword |
|---|
| 127 | 126 | # processing |
|---|
| 128 | | - sub bufp, len # update buffer length |
|---|
| 127 | + sub %bufp, len # update buffer length |
|---|
| 129 | 128 | align_loop: |
|---|
| 130 | 129 | crc32b %bl, crc_init_dw # compute crc32 of 1-byte |
|---|
| 131 | 130 | shr $8, tmp # get next byte |
|---|
| 132 | | - dec bufp |
|---|
| 131 | + dec %bufp |
|---|
| 133 | 132 | jne align_loop |
|---|
| 134 | 133 | |
|---|
| 135 | 134 | proc_block: |
|---|
| .. | .. |
|---|
| 169 | 168 | xor crc2, crc2 |
|---|
| 170 | 169 | |
|---|
| 171 | 170 | ## branch into array |
|---|
| 172 | | - lea jump_table(%rip), bufp |
|---|
| 173 | | - movzwq (bufp, %rax, 2), len |
|---|
| 174 | | - lea crc_array(%rip), bufp |
|---|
| 175 | | - lea (bufp, len, 1), bufp |
|---|
| 171 | + lea jump_table(%rip), %bufp |
|---|
| 172 | + movzwq (%bufp, %rax, 2), len |
|---|
| 173 | + lea crc_array(%rip), %bufp |
|---|
| 174 | + lea (%bufp, len, 1), %bufp |
|---|
| 176 | 175 | JMP_NOSPEC bufp |
|---|
| 177 | 176 | |
|---|
| 178 | 177 | ################################################################ |
|---|
| .. | .. |
|---|
| 218 | 217 | ## 4) Combine three results: |
|---|
| 219 | 218 | ################################################################ |
|---|
| 220 | 219 | |
|---|
| 221 | | - lea (K_table-8)(%rip), bufp # first entry is for idx 1 |
|---|
| 220 | + lea (K_table-8)(%rip), %bufp # first entry is for idx 1 |
|---|
| 222 | 221 | shlq $3, %rax # rax *= 8 |
|---|
| 223 | | - pmovzxdq (bufp,%rax), %xmm0 # 2 consts: K1:K2 |
|---|
| 222 | + pmovzxdq (%bufp,%rax), %xmm0 # 2 consts: K1:K2 |
|---|
| 224 | 223 | leal (%eax,%eax,2), %eax # rax *= 3 (total *24) |
|---|
| 225 | 224 | subq %rax, tmp # tmp -= rax*24 |
|---|
| 226 | 225 | |
|---|
| 227 | 226 | movq crc_init, %xmm1 # CRC for block 1 |
|---|
| 228 | | - PCLMULQDQ 0x00,%xmm0,%xmm1 # Multiply by K2 |
|---|
| 227 | + pclmulqdq $0x00, %xmm0, %xmm1 # Multiply by K2 |
|---|
| 229 | 228 | |
|---|
| 230 | 229 | movq crc1, %xmm2 # CRC for block 2 |
|---|
| 231 | | - PCLMULQDQ 0x10, %xmm0, %xmm2 # Multiply by K1 |
|---|
| 230 | + pclmulqdq $0x10, %xmm0, %xmm2 # Multiply by K1 |
|---|
| 232 | 231 | |
|---|
| 233 | 232 | pxor %xmm2,%xmm1 |
|---|
| 234 | 233 | movq %xmm1, %rax |
|---|
| .. | .. |
|---|
| 310 | 309 | popq %rsi |
|---|
| 311 | 310 | popq %rdi |
|---|
| 312 | 311 | popq %rbx |
|---|
| 313 | | - ret |
|---|
| 314 | | -ENDPROC(crc_pcl) |
|---|
| 312 | + RET |
|---|
| 313 | +SYM_FUNC_END(crc_pcl) |
|---|
| 315 | 314 | |
|---|
| 316 | 315 | .section .rodata, "a", @progbits |
|---|
| 317 | 316 | ################################################################ |
|---|