.. | .. |
---|
9 | 9 | #include <linux/init.h> |
---|
10 | 10 | #include <linux/kernel.h> |
---|
11 | 11 | #include <linux/list.h> |
---|
| 12 | +#include <linux/minmax.h> |
---|
12 | 13 | #include <linux/moduleparam.h> |
---|
13 | 14 | #include <linux/percpu.h> |
---|
14 | 15 | #include <linux/preempt.h> |
---|
15 | 16 | #include <linux/sched.h> |
---|
| 17 | +#include <linux/string.h> |
---|
16 | 18 | #include <linux/uaccess.h> |
---|
17 | 19 | |
---|
18 | 20 | #include "atomic.h" |
---|
.. | .. |
---|
1033 | 1035 | DEFINE_TSAN_ATOMIC_OPS(8); |
---|
1034 | 1036 | DEFINE_TSAN_ATOMIC_OPS(16); |
---|
1035 | 1037 | DEFINE_TSAN_ATOMIC_OPS(32); |
---|
| 1038 | +#ifdef CONFIG_64BIT |
---|
1036 | 1039 | DEFINE_TSAN_ATOMIC_OPS(64); |
---|
| 1040 | +#endif |
---|
1037 | 1041 | |
---|
1038 | 1042 | void __tsan_atomic_thread_fence(int memorder); |
---|
1039 | 1043 | void __tsan_atomic_thread_fence(int memorder) |
---|
.. | .. |
---|
1045 | 1049 | void __tsan_atomic_signal_fence(int memorder); |
---|
1046 | 1050 | void __tsan_atomic_signal_fence(int memorder) { } |
---|
1047 | 1051 | EXPORT_SYMBOL(__tsan_atomic_signal_fence); |
---|
| 1052 | + |
---|
| 1053 | +#ifdef __HAVE_ARCH_MEMSET |
---|
| 1054 | +void *__tsan_memset(void *s, int c, size_t count); |
---|
| 1055 | +noinline void *__tsan_memset(void *s, int c, size_t count) |
---|
| 1056 | +{ |
---|
| 1057 | + /* |
---|
| 1058 | + * Instead of not setting up watchpoints where accessed size is greater |
---|
| 1059 | + * than MAX_ENCODABLE_SIZE, truncate checked size to MAX_ENCODABLE_SIZE. |
---|
| 1060 | + */ |
---|
| 1061 | + size_t check_len = min_t(size_t, count, MAX_ENCODABLE_SIZE); |
---|
| 1062 | + |
---|
| 1063 | + check_access(s, check_len, KCSAN_ACCESS_WRITE); |
---|
| 1064 | + return memset(s, c, count); |
---|
| 1065 | +} |
---|
| 1066 | +#else |
---|
| 1067 | +void *__tsan_memset(void *s, int c, size_t count) __alias(memset); |
---|
| 1068 | +#endif |
---|
| 1069 | +EXPORT_SYMBOL(__tsan_memset); |
---|
| 1070 | + |
---|
| 1071 | +#ifdef __HAVE_ARCH_MEMMOVE |
---|
| 1072 | +void *__tsan_memmove(void *dst, const void *src, size_t len); |
---|
| 1073 | +noinline void *__tsan_memmove(void *dst, const void *src, size_t len) |
---|
| 1074 | +{ |
---|
| 1075 | + size_t check_len = min_t(size_t, len, MAX_ENCODABLE_SIZE); |
---|
| 1076 | + |
---|
| 1077 | + check_access(dst, check_len, KCSAN_ACCESS_WRITE); |
---|
| 1078 | + check_access(src, check_len, 0); |
---|
| 1079 | + return memmove(dst, src, len); |
---|
| 1080 | +} |
---|
| 1081 | +#else |
---|
| 1082 | +void *__tsan_memmove(void *dst, const void *src, size_t len) __alias(memmove); |
---|
| 1083 | +#endif |
---|
| 1084 | +EXPORT_SYMBOL(__tsan_memmove); |
---|
| 1085 | + |
---|
| 1086 | +#ifdef __HAVE_ARCH_MEMCPY |
---|
| 1087 | +void *__tsan_memcpy(void *dst, const void *src, size_t len); |
---|
| 1088 | +noinline void *__tsan_memcpy(void *dst, const void *src, size_t len) |
---|
| 1089 | +{ |
---|
| 1090 | + size_t check_len = min_t(size_t, len, MAX_ENCODABLE_SIZE); |
---|
| 1091 | + |
---|
| 1092 | + check_access(dst, check_len, KCSAN_ACCESS_WRITE); |
---|
| 1093 | + check_access(src, check_len, 0); |
---|
| 1094 | + return memcpy(dst, src, len); |
---|
| 1095 | +} |
---|
| 1096 | +#else |
---|
| 1097 | +void *__tsan_memcpy(void *dst, const void *src, size_t len) __alias(memcpy); |
---|
| 1098 | +#endif |
---|
| 1099 | +EXPORT_SYMBOL(__tsan_memcpy); |
---|