forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2024-09-20 a36159eec6ca17402b0e146b86efaf76568dc353 
 kernel/arch/mips/include/asm/barrier.h
....@@ -9,117 +9,26 @@
99 #define __ASM_BARRIER_H
1010 
1111 #include <asm/addrspace.h>
12
+#include <asm/sync.h>
1213 
13
-/*
14
- * Sync types defined by the MIPS architecture (document MD00087 table 6.5)
15
- * These values are used with the sync instruction to perform memory barriers.
16
- * Types of ordering guarantees available through the SYNC instruction:
17
- * - Completion Barriers
18
- * - Ordering Barriers
19
- * As compared to the completion barrier, the ordering barrier is a
20
- * lighter-weight operation as it does not require the specified instructions
21
- * before the SYNC to be already completed. Instead it only requires that those
22
- * specified instructions which are subsequent to the SYNC in the instruction
23
- * stream are never re-ordered for processing ahead of the specified
24
- * instructions which are before the SYNC in the instruction stream.
25
- * This potentially reduces how many cycles the barrier instruction must stall
26
- * before it completes.
27
- * Implementations that do not use any of the non-zero values of stype to define
28
- * different barriers, such as ordering barriers, must make those stype values
29
- * act the same as stype zero.
30
- */
14
+static inline void __sync(void)
15
+{
16
+	asm volatile(__SYNC(full, always) ::: "memory");
17
+}
3118 
32
-/*
33
- * Completion barriers:
34
- * - Every synchronizable specified memory instruction (loads or stores or both)
35
- *   that occurs in the instruction stream before the SYNC instruction must be
36
- *   already globally performed before any synchronizable specified memory
37
- *   instructions that occur after the SYNC are allowed to be performed, with
38
- *   respect to any other processor or coherent I/O module.
39
- *
40
- * - The barrier does not guarantee the order in which instruction fetches are
41
- *   performed.
42
- *
43
- * - A stype value of zero will always be defined such that it performs the most
44
- *   complete set of synchronization operations that are defined.This means
45
- *   stype zero always does a completion barrier that affects both loads and
46
- *   stores preceding the SYNC instruction and both loads and stores that are
47
- *   subsequent to the SYNC instruction. Non-zero values of stype may be defined
48
- *   by the architecture or specific implementations to perform synchronization
49
- *   behaviors that are less complete than that of stype zero. If an
50
- *   implementation does not use one of these non-zero values to define a
51
- *   different synchronization behavior, then that non-zero value of stype must
52
- *   act the same as stype zero completion barrier. This allows software written
53
- *   for an implementation with a lighter-weight barrier to work on another
54
- *   implementation which only implements the stype zero completion barrier.
55
- *
56
- * - A completion barrier is required, potentially in conjunction with SSNOP (in
57
- *   Release 1 of the Architecture) or EHB (in Release 2 of the Architecture),
58
- *   to guarantee that memory reference results are visible across operating
59
- *   mode changes. For example, a completion barrier is required on some
60
- *   implementations on entry to and exit from Debug Mode to guarantee that
61
- *   memory effects are handled correctly.
62
- */
19
+static inline void rmb(void)
20
+{
21
+	asm volatile(__SYNC(rmb, always) ::: "memory");
22
+}
23
+#define rmb rmb
6324 
64
-/*
65
- * stype 0 - A completion barrier that affects preceding loads and stores and
66
- * subsequent loads and stores.
67
- * Older instructions which must reach the load/store ordering point before the
68
- * SYNC instruction completes: Loads, Stores
69
- * Younger instructions which must reach the load/store ordering point only
70
- * after the SYNC instruction completes: Loads, Stores
71
- * Older instructions which must be globally performed when the SYNC instruction
72
- * completes: Loads, Stores
73
- */
74
-#define STYPE_SYNC 0x0
25
+static inline void wmb(void)
26
+{
27
+	asm volatile(__SYNC(wmb, always) ::: "memory");
28
+}
29
+#define wmb wmb
7530 
76
-/*
77
- * Ordering barriers:
78
- * - Every synchronizable specified memory instruction (loads or stores or both)
79
- *   that occurs in the instruction stream before the SYNC instruction must
80
- *   reach a stage in the load/store datapath after which no instruction
81
- *   re-ordering is possible before any synchronizable specified memory
82
- *   instruction which occurs after the SYNC instruction in the instruction
83
- *   stream reaches the same stage in the load/store datapath.
84
- *
85
- * - If any memory instruction before the SYNC instruction in program order,
86
- *   generates a memory request to the external memory and any memory
87
- *   instruction after the SYNC instruction in program order also generates a
88
- *   memory request to external memory, the memory request belonging to the
89
- *   older instruction must be globally performed before the time the memory
90
- *   request belonging to the younger instruction is globally performed.
91
- *
92
- * - The barrier does not guarantee the order in which instruction fetches are
93
- *   performed.
94
- */
95
-
96
-/*
97
- * stype 0x10 - An ordering barrier that affects preceding loads and stores and
98
- * subsequent loads and stores.
99
- * Older instructions which must reach the load/store ordering point before the
100
- * SYNC instruction completes: Loads, Stores
101
- * Younger instructions which must reach the load/store ordering point only
102
- * after the SYNC instruction completes: Loads, Stores
103
- * Older instructions which must be globally performed when the SYNC instruction
104
- * completes: N/A
105
- */
106
-#define STYPE_SYNC_MB 0x10
107
-
108
-
109
-#ifdef CONFIG_CPU_HAS_SYNC
110
-#define __sync()				\
111
-	__asm__ __volatile__(			\
112
-		".set	push\n\t"		\
113
-		".set	noreorder\n\t"		\
114
-		".set	mips2\n\t"		\
115
-		"sync\n\t"			\
116
-		".set	pop"			\
117
-		: /* no output */		\
118
-		: /* no input */		\
119
-		: "memory")
120
-#else
121
-#define __sync()	do { } while(0)
122
-#endif
31
+#define fast_mb()	__sync()
12332 
12433 #define __fast_iob()				\
12534 	__asm__ __volatile__(			\
....@@ -132,17 +41,8 @@
13241 		: "m" (*(int *)CKSEG1)		\
13342 		: "memory")
13443 #ifdef CONFIG_CPU_CAVIUM_OCTEON
135
-# define OCTEON_SYNCW_STR	".set push\n.set arch=octeon\nsyncw\nsyncw\n.set pop\n"
136
-# define __syncw()	__asm__ __volatile__(OCTEON_SYNCW_STR : : : "memory")
137
-
138
-# define fast_wmb()	__syncw()
139
-# define fast_rmb()	barrier()
140
-# define fast_mb()	__sync()
14144 # define fast_iob()	do { } while (0)
14245 #else /* ! CONFIG_CPU_CAVIUM_OCTEON */
143
-# define fast_wmb()	__sync()
144
-# define fast_rmb()	__sync()
145
-# define fast_mb()	__sync()
14646 # ifdef CONFIG_SGI_IP28
14747 #  define fast_iob()				\
14848 	__asm__ __volatile__(			\
....@@ -178,32 +78,32 @@
17878 
17979 #endif /* !CONFIG_CPU_HAS_WB */
18080 
181
-#define wmb()		fast_wmb()
182
-#define rmb()		fast_rmb()
183
-
18481 #if defined(CONFIG_WEAK_ORDERING)
185
-# ifdef CONFIG_CPU_CAVIUM_OCTEON
186
-#  define __smp_mb()	__sync()
187
-#  define __smp_rmb()	barrier()
188
-#  define __smp_wmb()	__syncw()
189
-# else
190
-#  define __smp_mb()	__asm__ __volatile__("sync" : : :"memory")
191
-#  define __smp_rmb()	__asm__ __volatile__("sync" : : :"memory")
192
-#  define __smp_wmb()	__asm__ __volatile__("sync" : : :"memory")
193
-# endif
82
+# define __smp_mb()	__sync()
83
+# define __smp_rmb()	rmb()
84
+# define __smp_wmb()	wmb()
19485 #else
195
-#define __smp_mb()	barrier()
196
-#define __smp_rmb()	barrier()
197
-#define __smp_wmb()	barrier()
86
+# define __smp_mb()	barrier()
87
+# define __smp_rmb()	barrier()
88
+# define __smp_wmb()	barrier()
19889 #endif
19990 
91
+/*
92
+ * When LL/SC does imply order, it must also be a compiler barrier to avoid the
93
+ * compiler from reordering where the CPU will not. When it does not imply
94
+ * order, the compiler is also free to reorder across the LL/SC loop and
95
+ * ordering will be done by smp_llsc_mb() and friends.
96
+ */
20097 #if defined(CONFIG_WEAK_REORDERING_BEYOND_LLSC) && defined(CONFIG_SMP)
201
-#define __WEAK_LLSC_MB		"	sync	\n"
98
+# define __WEAK_LLSC_MB		sync
99
+# define smp_llsc_mb() \
100
+	__asm__ __volatile__(__stringify(__WEAK_LLSC_MB) : : :"memory")
101
+# define __LLSC_CLOBBER
202102 #else
203
-#define __WEAK_LLSC_MB		"		\n"
103
+# define __WEAK_LLSC_MB
104
+# define smp_llsc_mb()		do { } while (0)
105
+# define __LLSC_CLOBBER		"memory"
204106 #endif
205
-
206
-#define smp_llsc_mb()	__asm__ __volatile__(__WEAK_LLSC_MB : : :"memory")
207107 
208108 #ifdef CONFIG_CPU_CAVIUM_OCTEON
209109 #define smp_mb__before_llsc() smp_wmb()
....@@ -219,9 +119,24 @@
219119 #define nudge_writes() mb()
220120 #endif
221121 
222
-#define __smp_mb__before_atomic()	__smp_mb__before_llsc()
122
+/*
123
+ * In the Loongson3 LL/SC workaround case, all of our LL/SC loops already have
124
+ * a completion barrier immediately preceding the LL instruction. Therefore we
125
+ * can skip emitting a barrier from __smp_mb__before_atomic().
126
+ */
127
+#ifdef CONFIG_CPU_LOONGSON3_WORKAROUNDS
128
+# define __smp_mb__before_atomic()
129
+#else
130
+# define __smp_mb__before_atomic()	__smp_mb__before_llsc()
131
+#endif
132
+
223133 #define __smp_mb__after_atomic()	smp_llsc_mb()
224134 
135
+static inline void sync_ginv(void)
136
+{
137
+	asm volatile(__SYNC(ginv, always));
138
+}
139
+
225140 #include <asm-generic/barrier.h>
226141 
227142 #endif /* __ASM_BARRIER_H */