forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2023-12-09 95099d4622f8cb224d94e314c7a8e0df60b13f87 
 kernel/arch/arm64/include/asm/atomic_ll_sc.h
....@@ -1,103 +1,101 @@
1
+/* SPDX-License-Identifier: GPL-2.0-only */
12 /*
23  * Based on arch/arm/include/asm/atomic.h
34  *
45  * Copyright (C) 1996 Russell King.
56  * Copyright (C) 2002 Deep Blue Solutions Ltd.
67  * Copyright (C) 2012 ARM Ltd.
7
- *
8
- * This program is free software; you can redistribute it and/or modify
9
- * it under the terms of the GNU General Public License version 2 as
10
- * published by the Free Software Foundation.
11
- *
12
- * This program is distributed in the hope that it will be useful,
13
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
14
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15
- * GNU General Public License for more details.
16
- *
17
- * You should have received a copy of the GNU General Public License
18
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
198  */
209 
2110 #ifndef __ASM_ATOMIC_LL_SC_H
2211 #define __ASM_ATOMIC_LL_SC_H
2312 
24
-#ifndef __ARM64_IN_ATOMIC_IMPL
25
-#error "please don't include this file directly"
13
+#include <linux/stringify.h>
14
+
15
+#ifdef CONFIG_ARM64_LSE_ATOMICS
16
+#define __LL_SC_FALLBACK(asm_ops)					\
17
+"	b	3f\n"							\
18
+"	.subsection	1\n"						\
19
+"3:\n"									\
20
+asm_ops "\n"								\
21
+"	b	4f\n"							\
22
+"	.previous\n"							\
23
+"4:\n"
24
+#else
25
+#define __LL_SC_FALLBACK(asm_ops) asm_ops
26
+#endif
27
+
28
+#ifndef CONFIG_CC_HAS_K_CONSTRAINT
29
+#define K
2630 #endif
2731 
2832 /*
2933  * AArch64 UP and SMP safe atomic ops.  We use load exclusive and
3034  * store exclusive to ensure that these are atomic.  We may loop
3135  * to ensure that the update happens.
32
- *
33
- * NOTE: these functions do *not* follow the PCS and must explicitly
34
- * save any clobbered registers other than x0 (regardless of return
35
- * value).  This is achieved through -fcall-saved-* compiler flags for
36
- * this file, which unfortunately don't work on a per-function basis
37
- * (the optimize attribute silently ignores these options).
3836  */
3937 
4038 #define ATOMIC_OP(op, asm_op, constraint)				\
41
-__LL_SC_INLINE void							\
42
-__LL_SC_PREFIX(atomic_##op(int i, atomic_t *v))				\
39
+static inline void							\
40
+__ll_sc_atomic_##op(int i, atomic_t *v)					\
4341 {									\
4442 	unsigned long tmp;						\
4543 	int result;							\
4644 									\
4745 	asm volatile("// atomic_" #op "\n"				\
46
+	__LL_SC_FALLBACK(						\
4847 "	prfm	pstl1strm, %2\n"					\
4948 "1:	ldxr	%w0, %2\n"						\
5049 "	" #asm_op "	%w0, %w0, %w3\n"				\
5150 "	stxr	%w1, %w0, %2\n"						\
52
-"	cbnz	%w1, 1b"						\
51
+"	cbnz	%w1, 1b\n")						\
5352 	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)		\
54
-	: #constraint "r" (i));						\
55
-}									\
56
-__LL_SC_EXPORT(atomic_##op);
53
+	: __stringify(constraint) "r" (i));				\
54
+}
5755 
5856 #define ATOMIC_OP_RETURN(name, mb, acq, rel, cl, op, asm_op, constraint)\
59
-__LL_SC_INLINE int							\
60
-__LL_SC_PREFIX(atomic_##op##_return##name(int i, atomic_t *v))		\
57
+static inline int							\
58
+__ll_sc_atomic_##op##_return##name(int i, atomic_t *v)			\
6159 {									\
6260 	unsigned long tmp;						\
6361 	int result;							\
6462 									\
6563 	asm volatile("// atomic_" #op "_return" #name "\n"		\
64
+	__LL_SC_FALLBACK(						\
6665 "	prfm	pstl1strm, %2\n"					\
6766 "1:	ld" #acq "xr	%w0, %2\n"					\
6867 "	" #asm_op "	%w0, %w0, %w3\n"				\
6968 "	st" #rel "xr	%w1, %w0, %2\n"					\
7069 "	cbnz	%w1, 1b\n"						\
71
-"	" #mb								\
70
+"	" #mb )								\
7271 	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)		\
73
-	: #constraint "r" (i)						\
72
+	: __stringify(constraint) "r" (i)				\
7473 	: cl);								\
7574 									\
7675 	return result;							\
77
-}									\
78
-__LL_SC_EXPORT(atomic_##op##_return##name);
76
+}
7977 
80
-#define ATOMIC_FETCH_OP(name, mb, acq, rel, cl, op, asm_op, constraint)	\
81
-__LL_SC_INLINE int							\
82
-__LL_SC_PREFIX(atomic_fetch_##op##name(int i, atomic_t *v))		\
78
+#define ATOMIC_FETCH_OP(name, mb, acq, rel, cl, op, asm_op, constraint) \
79
+static inline int							\
80
+__ll_sc_atomic_fetch_##op##name(int i, atomic_t *v)			\
8381 {									\
8482 	unsigned long tmp;						\
8583 	int val, result;						\
8684 									\
8785 	asm volatile("// atomic_fetch_" #op #name "\n"			\
86
+	__LL_SC_FALLBACK(						\
8887 "	prfm	pstl1strm, %3\n"					\
8988 "1:	ld" #acq "xr	%w0, %3\n"					\
9089 "	" #asm_op "	%w1, %w0, %w4\n"				\
9190 "	st" #rel "xr	%w2, %w1, %3\n"					\
9291 "	cbnz	%w2, 1b\n"						\
93
-"	" #mb								\
92
+"	" #mb )								\
9493 	: "=&r" (result), "=&r" (val), "=&r" (tmp), "+Q" (v->counter)	\
95
-	: #constraint "r" (i)						\
94
+	: __stringify(constraint) "r" (i)				\
9695 	: cl);								\
9796 									\
9897 	return result;							\
99
-}									\
100
-__LL_SC_EXPORT(atomic_fetch_##op##name);
98
+}
10199 
102100 #define ATOMIC_OPS(...)							\
103101 	ATOMIC_OP(__VA_ARGS__)						\
....@@ -121,10 +119,15 @@
121119 	ATOMIC_FETCH_OP (_acquire,        , a,  , "memory", __VA_ARGS__)\
122120 	ATOMIC_FETCH_OP (_release,        ,  , l, "memory", __VA_ARGS__)
123121 
124
-ATOMIC_OPS(and, and, )
122
+ATOMIC_OPS(and, and, K)
123
+ATOMIC_OPS(or, orr, K)
124
+ATOMIC_OPS(xor, eor, K)
125
+/*
126
+ * GAS converts the mysterious and undocumented BIC (immediate) alias to
127
+ * an AND (immediate) instruction with the immediate inverted. We don't
128
+ * have a constraint for this, so fall back to register.
129
+ */
125130 ATOMIC_OPS(andnot, bic, )
126
-ATOMIC_OPS(or, orr, )
127
-ATOMIC_OPS(xor, eor, )
128131 
129132 #undef ATOMIC_OPS
130133 #undef ATOMIC_FETCH_OP
....@@ -132,66 +135,66 @@
132135 #undef ATOMIC_OP
133136 
134137 #define ATOMIC64_OP(op, asm_op, constraint)				\
135
-__LL_SC_INLINE void							\
136
-__LL_SC_PREFIX(atomic64_##op(long i, atomic64_t *v))			\
138
+static inline void							\
139
+__ll_sc_atomic64_##op(s64 i, atomic64_t *v)				\
137140 {									\
138
-	long result;							\
141
+	s64 result;							\
139142 	unsigned long tmp;						\
140143 									\
141144 	asm volatile("// atomic64_" #op "\n"				\
145
+	__LL_SC_FALLBACK(						\
142146 "	prfm	pstl1strm, %2\n"					\
143147 "1:	ldxr	%0, %2\n"						\
144148 "	" #asm_op "	%0, %0, %3\n"					\
145149 "	stxr	%w1, %0, %2\n"						\
146
-"	cbnz	%w1, 1b"						\
150
+"	cbnz	%w1, 1b")						\
147151 	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)		\
148
-	: #constraint "r" (i));						\
149
-}									\
150
-__LL_SC_EXPORT(atomic64_##op);
152
+	: __stringify(constraint) "r" (i));				\
153
+}
151154 
152155 #define ATOMIC64_OP_RETURN(name, mb, acq, rel, cl, op, asm_op, constraint)\
153
-__LL_SC_INLINE long							\
154
-__LL_SC_PREFIX(atomic64_##op##_return##name(long i, atomic64_t *v))	\
156
+static inline long							\
157
+__ll_sc_atomic64_##op##_return##name(s64 i, atomic64_t *v)		\
155158 {									\
156
-	long result;							\
159
+	s64 result;							\
157160 	unsigned long tmp;						\
158161 									\
159162 	asm volatile("// atomic64_" #op "_return" #name "\n"		\
163
+	__LL_SC_FALLBACK(						\
160164 "	prfm	pstl1strm, %2\n"					\
161165 "1:	ld" #acq "xr	%0, %2\n"					\
162166 "	" #asm_op "	%0, %0, %3\n"					\
163167 "	st" #rel "xr	%w1, %0, %2\n"					\
164168 "	cbnz	%w1, 1b\n"						\
165
-"	" #mb								\
169
+"	" #mb )								\
166170 	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)		\
167
-	: #constraint "r" (i)						\
171
+	: __stringify(constraint) "r" (i)				\
168172 	: cl);								\
169173 									\
170174 	return result;							\
171
-}									\
172
-__LL_SC_EXPORT(atomic64_##op##_return##name);
175
+}
173176 
174177 #define ATOMIC64_FETCH_OP(name, mb, acq, rel, cl, op, asm_op, constraint)\
175
-__LL_SC_INLINE long							\
176
-__LL_SC_PREFIX(atomic64_fetch_##op##name(long i, atomic64_t *v))	\
178
+static inline long							\
179
+__ll_sc_atomic64_fetch_##op##name(s64 i, atomic64_t *v)		\
177180 {									\
178
-	long result, val;						\
181
+	s64 result, val;						\
179182 	unsigned long tmp;						\
180183 									\
181184 	asm volatile("// atomic64_fetch_" #op #name "\n"		\
185
+	__LL_SC_FALLBACK(						\
182186 "	prfm	pstl1strm, %3\n"					\
183187 "1:	ld" #acq "xr	%0, %3\n"					\
184188 "	" #asm_op "	%1, %0, %4\n"					\
185189 "	st" #rel "xr	%w2, %1, %3\n"					\
186190 "	cbnz	%w2, 1b\n"						\
187
-"	" #mb								\
191
+"	" #mb )								\
188192 	: "=&r" (result), "=&r" (val), "=&r" (tmp), "+Q" (v->counter)	\
189
-	: #constraint "r" (i)						\
193
+	: __stringify(constraint) "r" (i)				\
190194 	: cl);								\
191195 									\
192196 	return result;							\
193
-}									\
194
-__LL_SC_EXPORT(atomic64_fetch_##op##name);
197
+}
195198 
196199 #define ATOMIC64_OPS(...)						\
197200 	ATOMIC64_OP(__VA_ARGS__)					\
....@@ -216,22 +219,28 @@
216219 	ATOMIC64_FETCH_OP (_release,,  , l, "memory", __VA_ARGS__)
217220 
218221 ATOMIC64_OPS(and, and, L)
219
-ATOMIC64_OPS(andnot, bic, )
220222 ATOMIC64_OPS(or, orr, L)
221223 ATOMIC64_OPS(xor, eor, L)
224
+/*
225
+ * GAS converts the mysterious and undocumented BIC (immediate) alias to
226
+ * an AND (immediate) instruction with the immediate inverted. We don't
227
+ * have a constraint for this, so fall back to register.
228
+ */
229
+ATOMIC64_OPS(andnot, bic, )
222230 
223231 #undef ATOMIC64_OPS
224232 #undef ATOMIC64_FETCH_OP
225233 #undef ATOMIC64_OP_RETURN
226234 #undef ATOMIC64_OP
227235 
228
-__LL_SC_INLINE long
229
-__LL_SC_PREFIX(atomic64_dec_if_positive(atomic64_t *v))
236
+static inline s64
237
+__ll_sc_atomic64_dec_if_positive(atomic64_t *v)
230238 {
231
-	long result;
239
+	s64 result;
232240 	unsigned long tmp;
233241 
234242 	asm volatile("// atomic64_dec_if_positive\n"
243
+	__LL_SC_FALLBACK(
235244 "	prfm	pstl1strm, %2\n"
236245 "1:	ldxr	%0, %2\n"
237246 "	subs	%0, %0, #1\n"
....@@ -239,76 +248,85 @@
239248 "	stlxr	%w1, %0, %2\n"
240249 "	cbnz	%w1, 1b\n"
241250 "	dmb	ish\n"
242
-"2:"
251
+"2:")
243252 	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
244253 	:
245254 	: "cc", "memory");
246255 
247256 	return result;
248257 }
249
-__LL_SC_EXPORT(atomic64_dec_if_positive);
250258 
251
-#define __CMPXCHG_CASE(w, sz, name, mb, acq, rel, cl, constraint)	\
252
-__LL_SC_INLINE unsigned long						\
253
-__LL_SC_PREFIX(__cmpxchg_case_##name(volatile void *ptr,		\
254
-				     unsigned long old,			\
255
-				     unsigned long new))		\
259
+#define __CMPXCHG_CASE(w, sfx, name, sz, mb, acq, rel, cl, constraint)	\
260
+static inline u##sz							\
261
+__ll_sc__cmpxchg_case_##name##sz(volatile void *ptr,			\
262
+					 unsigned long old,		\
263
+					 u##sz new)			\
256264 {									\
257
-	unsigned long tmp, oldval;					\
265
+	unsigned long tmp;						\
266
+	u##sz oldval;							\
267
+									\
268
+	/*								\
269
+	 * Sub-word sizes require explicit casting so that the compare  \
270
+	 * part of the cmpxchg doesn't end up interpreting non-zero	\
271
+	 * upper bits of the register containing "old".			\
272
+	 */								\
273
+	if (sz < 32)							\
274
+		old = (u##sz)old;					\
258275 									\
259276 	asm volatile(							\
277
+	__LL_SC_FALLBACK(						\
260278 	"	prfm	pstl1strm, %[v]\n"				\
261
-	"1:	ld" #acq "xr" #sz "\t%" #w "[oldval], %[v]\n"		\
279
+	"1:	ld" #acq "xr" #sfx "\t%" #w "[oldval], %[v]\n"		\
262280 	"	eor	%" #w "[tmp], %" #w "[oldval], %" #w "[old]\n"	\
263281 	"	cbnz	%" #w "[tmp], 2f\n"				\
264
-	"	st" #rel "xr" #sz "\t%w[tmp], %" #w "[new], %[v]\n"	\
282
+	"	st" #rel "xr" #sfx "\t%w[tmp], %" #w "[new], %[v]\n"	\
265283 	"	cbnz	%w[tmp], 1b\n"					\
266284 	"	" #mb "\n"						\
267
-	"2:"								\
285
+	"2:")								\
268286 	: [tmp] "=&r" (tmp), [oldval] "=&r" (oldval),			\
269
-	  [v] "+Q" (*(unsigned long *)ptr)				\
270
-	: [old] #constraint "r" (old), [new] "r" (new)				\
287
+	  [v] "+Q" (*(u##sz *)ptr)					\
288
+	: [old] __stringify(constraint) "r" (old), [new] "r" (new)	\
271289 	: cl);								\
272290 									\
273291 	return oldval;							\
274
-}									\
275
-__LL_SC_EXPORT(__cmpxchg_case_##name);
292
+}
276293 
277294 /*
278295  * Earlier versions of GCC (no later than 8.1.0) appear to incorrectly
279296  * handle the 'K' constraint for the value 4294967295 - thus we use no
280297  * constraint for 32 bit operations.
281298  */
282
-__CMPXCHG_CASE(w, b,     1,        ,  ,  ,         , )
283
-__CMPXCHG_CASE(w, h,     2,        ,  ,  ,         , )
284
-__CMPXCHG_CASE(w,  ,     4,        ,  ,  ,         , )
285
-__CMPXCHG_CASE( ,  ,     8,        ,  ,  ,         , L)
286
-__CMPXCHG_CASE(w, b, acq_1,        , a,  , "memory", )
287
-__CMPXCHG_CASE(w, h, acq_2,        , a,  , "memory", )
288
-__CMPXCHG_CASE(w,  , acq_4,        , a,  , "memory", )
289
-__CMPXCHG_CASE( ,  , acq_8,        , a,  , "memory", L)
290
-__CMPXCHG_CASE(w, b, rel_1,        ,  , l, "memory", )
291
-__CMPXCHG_CASE(w, h, rel_2,        ,  , l, "memory", )
292
-__CMPXCHG_CASE(w,  , rel_4,        ,  , l, "memory", )
293
-__CMPXCHG_CASE( ,  , rel_8,        ,  , l, "memory", L)
294
-__CMPXCHG_CASE(w, b,  mb_1, dmb ish,  , l, "memory", )
295
-__CMPXCHG_CASE(w, h,  mb_2, dmb ish,  , l, "memory", )
296
-__CMPXCHG_CASE(w,  ,  mb_4, dmb ish,  , l, "memory", )
297
-__CMPXCHG_CASE( ,  ,  mb_8, dmb ish,  , l, "memory", L)
299
+__CMPXCHG_CASE(w, b,     ,  8,        ,  ,  ,         , K)
300
+__CMPXCHG_CASE(w, h,     , 16,        ,  ,  ,         , K)
301
+__CMPXCHG_CASE(w,  ,     , 32,        ,  ,  ,         , K)
302
+__CMPXCHG_CASE( ,  ,     , 64,        ,  ,  ,         , L)
303
+__CMPXCHG_CASE(w, b, acq_,  8,        , a,  , "memory", K)
304
+__CMPXCHG_CASE(w, h, acq_, 16,        , a,  , "memory", K)
305
+__CMPXCHG_CASE(w,  , acq_, 32,        , a,  , "memory", K)
306
+__CMPXCHG_CASE( ,  , acq_, 64,        , a,  , "memory", L)
307
+__CMPXCHG_CASE(w, b, rel_,  8,        ,  , l, "memory", K)
308
+__CMPXCHG_CASE(w, h, rel_, 16,        ,  , l, "memory", K)
309
+__CMPXCHG_CASE(w,  , rel_, 32,        ,  , l, "memory", K)
310
+__CMPXCHG_CASE( ,  , rel_, 64,        ,  , l, "memory", L)
311
+__CMPXCHG_CASE(w, b,  mb_,  8, dmb ish,  , l, "memory", K)
312
+__CMPXCHG_CASE(w, h,  mb_, 16, dmb ish,  , l, "memory", K)
313
+__CMPXCHG_CASE(w,  ,  mb_, 32, dmb ish,  , l, "memory", K)
314
+__CMPXCHG_CASE( ,  ,  mb_, 64, dmb ish,  , l, "memory", L)
298315 
299316 #undef __CMPXCHG_CASE
300317 
301318 #define __CMPXCHG_DBL(name, mb, rel, cl)				\
302
-__LL_SC_INLINE long							\
303
-__LL_SC_PREFIX(__cmpxchg_double##name(unsigned long old1,		\
319
+static inline long							\
320
+__ll_sc__cmpxchg_double##name(unsigned long old1,			\
304321 				      unsigned long old2,		\
305322 				      unsigned long new1,		\
306323 				      unsigned long new2,		\
307
-				      volatile void *ptr))		\
324
+				      volatile void *ptr)		\
308325 {									\
309326 	unsigned long tmp, ret;						\
310327 									\
311328 	asm volatile("// __cmpxchg_double" #name "\n"			\
329
+	__LL_SC_FALLBACK(						\
312330 	"	prfm	pstl1strm, %2\n"				\
313331 	"1:	ldxp	%0, %1, %2\n"					\
314332 	"	eor	%0, %0, %3\n"					\
....@@ -318,18 +336,18 @@
318336 	"	st" #rel "xp	%w0, %5, %6, %2\n"			\
319337 	"	cbnz	%w0, 1b\n"					\
320338 	"	" #mb "\n"						\
321
-	"2:"								\
339
+	"2:")								\
322340 	: "=&r" (tmp), "=&r" (ret), "+Q" (*(unsigned long *)ptr)	\
323341 	: "r" (old1), "r" (old2), "r" (new1), "r" (new2)		\
324342 	: cl);								\
325343 									\
326344 	return ret;							\
327
-}									\
328
-__LL_SC_EXPORT(__cmpxchg_double##name);
345
+}
329346 
330347 __CMPXCHG_DBL(   ,        ,  ,         )
331348 __CMPXCHG_DBL(_mb, dmb ish, l, "memory")
332349 
333350 #undef __CMPXCHG_DBL
351
+#undef K
334352 
335353 #endif	/* __ASM_ATOMIC_LL_SC_H */