| .. | .. |
|---|
| 8 | 8 | */ |
|---|
| 9 | 9 | #include <linux/linkage.h> |
|---|
| 10 | 10 | #include <asm/cpufeatures.h> |
|---|
| 11 | | -#include <asm/alternative-asm.h> |
|---|
| 11 | +#include <asm/alternative.h> |
|---|
| 12 | 12 | #include <asm/export.h> |
|---|
| 13 | 13 | |
|---|
| 14 | 14 | #undef memmove |
|---|
| .. | .. |
|---|
| 24 | 24 | * Output: |
|---|
| 25 | 25 | * rax: dest |
|---|
| 26 | 26 | */ |
|---|
| 27 | | -.weak memmove |
|---|
| 28 | | -.p2align 4, 0x90 |
|---|
| 29 | | -memmove: |
|---|
| 30 | | -ENTRY(__memmove) |
|---|
| 27 | +SYM_FUNC_START_WEAK(memmove) |
|---|
| 28 | +SYM_FUNC_START(__memmove) |
|---|
| 31 | 29 | |
|---|
| 32 | | - /* Handle more 32 bytes in loop */ |
|---|
| 33 | 30 | mov %rdi, %rax |
|---|
| 34 | | - cmp $0x20, %rdx |
|---|
| 35 | | - jb 1f |
|---|
| 36 | 31 | |
|---|
| 37 | 32 | /* Decide forward/backward copy mode */ |
|---|
| 38 | 33 | cmp %rdi, %rsi |
|---|
| .. | .. |
|---|
| 42 | 37 | cmp %rdi, %r8 |
|---|
| 43 | 38 | jg 2f |
|---|
| 44 | 39 | |
|---|
| 40 | + /* FSRM implies ERMS => no length checks, do the copy directly */ |
|---|
| 45 | 41 | .Lmemmove_begin_forward: |
|---|
| 46 | | - ALTERNATIVE "", "movq %rdx, %rcx; rep movsb; retq", X86_FEATURE_ERMS |
|---|
| 42 | + ALTERNATIVE "cmp $0x20, %rdx; jb 1f", "", X86_FEATURE_FSRM |
|---|
| 43 | + ALTERNATIVE "", "jmp .Lmemmove_erms", X86_FEATURE_ERMS |
|---|
| 47 | 44 | |
|---|
| 48 | 45 | /* |
|---|
| 49 | 46 | * movsq instruction have many startup latency |
|---|
| .. | .. |
|---|
| 114 | 111 | */ |
|---|
| 115 | 112 | .p2align 4 |
|---|
| 116 | 113 | 2: |
|---|
| 114 | + cmp $0x20, %rdx |
|---|
| 115 | + jb 1f |
|---|
| 117 | 116 | cmp $680, %rdx |
|---|
| 118 | 117 | jb 6f |
|---|
| 119 | 118 | cmp %dil, %sil |
|---|
| .. | .. |
|---|
| 206 | 205 | movb (%rsi), %r11b |
|---|
| 207 | 206 | movb %r11b, (%rdi) |
|---|
| 208 | 207 | 13: |
|---|
| 209 | | - retq |
|---|
| 210 | | -ENDPROC(__memmove) |
|---|
| 211 | | -ENDPROC(memmove) |
|---|
| 208 | + RET |
|---|
| 209 | + |
|---|
| 210 | +.Lmemmove_erms: |
|---|
| 211 | + movq %rdx, %rcx |
|---|
| 212 | + rep movsb |
|---|
| 213 | + RET |
|---|
| 214 | +SYM_FUNC_END(__memmove) |
|---|
| 215 | +SYM_FUNC_END_ALIAS(memmove) |
|---|
| 212 | 216 | EXPORT_SYMBOL(__memmove) |
|---|
| 213 | 217 | EXPORT_SYMBOL(memmove) |
|---|