write in 30M

hc

2024-02-20 ea08eeccae9297f7aabd2ef7f0c2517ac4549acc

 kernel/arch/x86/include/asm/string_64.h
..
..
@@ -7,24 +7,6 @@
7
7
 
8
8
 /* Written 2002 by Andi Kleen */
9
9
 
10
-/* Only used for special circumstances. Stolen from i386/string.h */
11
-static __always_inline void *__inline_memcpy(void *to, const void *from, size_t n)
12
-{
13
-	unsigned long d0, d1, d2;
14
-	asm volatile("rep ; movsl\n\t"
15
-		     "testb $2,%b4\n\t"
16
-		     "je 1f\n\t"
17
-		     "movsw\n"
18
-		     "1:\ttestb $1,%b4\n\t"
19
-		     "je 2f\n\t"
20
-		     "movsb\n"
21
-		     "2:"
22
-		     : "=&c" (d0), "=&D" (d1), "=&S" (d2)
23
-		     : "0" (n / 4), "q" (n), "1" ((long)to), "2" ((long)from)
24
-		     : "memory");
25
-	return to;
26
-}
27
-
28
10
 /* Even with __builtin_ the compiler may decide to use the out of line
29
11
    function. */
30
12
 
..
..
@@ -100,41 +82,27 @@
100
82
 
101
83
 #endif
102
84
 
103
-#define __HAVE_ARCH_MEMCPY_MCSAFE 1
104
-__must_check unsigned long __memcpy_mcsafe(void *dst, const void *src,
105
-		size_t cnt);
106
-DECLARE_STATIC_KEY_FALSE(mcsafe_key);
107
-
108
-/**
109
- * memcpy_mcsafe - copy memory with indication if a machine check happened
110
- *
111
- * @dst:	destination address
112
- * @src:	source address
113
- * @cnt:	number of bytes to copy
114
- *
115
- * Low level memory copy function that catches machine checks
116
- * We only call into the "safe" function on systems that can
117
- * actually do machine check recovery. Everyone else can just
118
- * use memcpy().
119
- *
120
- * Return 0 for success, or number of bytes not copied if there was an
121
- * exception.
122
- */
123
-static __always_inline __must_check unsigned long
124
-memcpy_mcsafe(void *dst, const void *src, size_t cnt)
125
-{
126
-#ifdef CONFIG_X86_MCE
127
-	if (static_branch_unlikely(&mcsafe_key))
128
-		return __memcpy_mcsafe(dst, src, cnt);
129
-	else
130
-#endif
131
-		memcpy(dst, src, cnt);
132
-	return 0;
133
-}
134
-
135
85
 #ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
136
86
 #define __HAVE_ARCH_MEMCPY_FLUSHCACHE 1
137
-void memcpy_flushcache(void *dst, const void *src, size_t cnt);
87
+void __memcpy_flushcache(void *dst, const void *src, size_t cnt);
88
+static __always_inline void memcpy_flushcache(void *dst, const void *src, size_t cnt)
89
+{
90
+	if (__builtin_constant_p(cnt)) {
91
+		switch (cnt) {
92
+			case 4:
93
+				asm ("movntil %1, %0" : "=m"(*(u32 *)dst) : "r"(*(u32 *)src));
94
+				return;
95
+			case 8:
96
+				asm ("movntiq %1, %0" : "=m"(*(u64 *)dst) : "r"(*(u64 *)src));
97
+				return;
98
+			case 16:
99
+				asm ("movntiq %1, %0" : "=m"(*(u64 *)dst) : "r"(*(u64 *)src));
100
+				asm ("movntiq %1, %0" : "=m"(*(u64 *)(dst + 8)) : "r"(*(u64 *)(src + 8)));
101
+				return;
102
+		}
103
+	}
104
+	__memcpy_flushcache(dst, src, cnt);
105
+}
138
106
 #endif
139
107
 
140
108
 #endif /* __KERNEL__ */