forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2024-02-20 102a0743326a03cd1a1202ceda21e175b7d3575c 
 kernel/tools/testing/selftests/rseq/rseq-s390.h
....@@ -1,6 +1,13 @@
11 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
22 
3
-#define RSEQ_SIG	0x53053053
3
+/*
4
+ * RSEQ_SIG uses the trap4 instruction. As Linux does not make use of the
5
+ * access-register mode nor the linkage stack this instruction will always
6
+ * cause a special-operation exception (the trap-enabled bit in the DUCT
7
+ * is and will stay 0). The instruction pattern is
8
+ *	b2 ff 0f ff	trap4	4095(%r0)
9
+ */
10
+#define RSEQ_SIG	0xB2FF0FFF
411 
512 #define rseq_smp_mb()	__asm__ __volatile__ ("bcr 15,0" ::: "memory")
613 #define rseq_smp_rmb()	rseq_smp_mb()
....@@ -37,22 +44,54 @@
3744 
3845 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,			\
3946 				start_ip, post_commit_offset, abort_ip)	\
40
-		".pushsection __rseq_table, \"aw\"\n\t"			\
47
+		".pushsection __rseq_cs, \"aw\"\n\t"			\
4148 		".balign 32\n\t"					\
4249 		__rseq_str(label) ":\n\t"				\
4350 		".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
4451 		".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
52
+		".popsection\n\t"					\
53
+		".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"		\
54
+		".quad " __rseq_str(label) "b\n\t"			\
55
+		".popsection\n\t"
56
+
57
+/*
58
+ * Exit points of a rseq critical section consist of all instructions outside
59
+ * of the critical section where a critical section can either branch to or
60
+ * reach through the normal course of its execution. The abort IP and the
61
+ * post-commit IP are already part of the __rseq_cs section and should not be
62
+ * explicitly defined as additional exit points. Knowing all exit points is
63
+ * useful to assist debuggers stepping over the critical section.
64
+ */
65
+#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)			\
66
+		".pushsection __rseq_exit_point_array, \"aw\"\n\t"	\
67
+		".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
4568 		".popsection\n\t"
4669 
4770 #elif __s390__
4871 
4972 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,			\
5073 				start_ip, post_commit_offset, abort_ip)	\
51
-		".pushsection __rseq_table, \"aw\"\n\t"			\
74
+		".pushsection __rseq_cs, \"aw\"\n\t"			\
5275 		".balign 32\n\t"					\
5376 		__rseq_str(label) ":\n\t"				\
5477 		".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
5578 		".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) "\n\t" \
79
+		".popsection\n\t"					\
80
+		".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"		\
81
+		".long 0x0, " __rseq_str(label) "b\n\t"			\
82
+		".popsection\n\t"
83
+
84
+/*
85
+ * Exit points of a rseq critical section consist of all instructions outside
86
+ * of the critical section where a critical section can either branch to or
87
+ * reach through the normal course of its execution. The abort IP and the
88
+ * post-commit IP are already part of the __rseq_cs section and should not be
89
+ * explicitly defined as additional exit points. Knowing all exit points is
90
+ * useful to assist debuggers stepping over the critical section.
91
+ */
92
+#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)			\
93
+		".pushsection __rseq_exit_point_array, \"aw\"\n\t"	\
94
+		".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) "\n\t" \
5695 		".popsection\n\t"
5796 
5897 #define LONG_L			"l"
....@@ -85,14 +124,14 @@
85124 		".long " __rseq_str(RSEQ_SIG) "\n\t"			\
86125 		__rseq_str(label) ":\n\t"				\
87126 		teardown						\
88
-		"j %l[" __rseq_str(abort_label) "]\n\t"			\
127
+		"jg %l[" __rseq_str(abort_label) "]\n\t"		\
89128 		".popsection\n\t"
90129 
91130 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label)		\
92131 		".pushsection __rseq_failure, \"ax\"\n\t"		\
93132 		__rseq_str(label) ":\n\t"				\
94133 		teardown						\
95
-		"j %l[" __rseq_str(cmpfail_label) "]\n\t"		\
134
+		"jg %l[" __rseq_str(cmpfail_label) "]\n\t"		\
96135 		".popsection\n\t"
97136 
98137 static inline __attribute__((always_inline))
....@@ -102,6 +141,11 @@
102141 
103142 	__asm__ __volatile__ goto (
104143 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
144
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
145
+#ifdef RSEQ_COMPARE_TWICE
146
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
147
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
148
+#endif
105149 		/* Start rseq by storing table entry pointer into rseq_cs. */
106150 		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
107151 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
....@@ -121,8 +165,8 @@
121165 		RSEQ_ASM_DEFINE_ABORT(4, "", abort)
122166 		: /* gcc asm goto does not allow outputs */
123167 		: [cpu_id]		"r" (cpu),
124
-		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
125
-		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
168
+		  [current_cpu_id]	"m" (rseq_get_abi()->cpu_id),
169
+		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
126170 		  [v]			"m" (*v),
127171 		  [expect]		"r" (expect),
128172 		  [newv]		"r" (newv)
....@@ -134,16 +178,21 @@
134178 		  , error1, error2
135179 #endif
136180 	);
181
+	rseq_after_asm_goto();
137182 	return 0;
138183 abort:
184
+	rseq_after_asm_goto();
139185 	RSEQ_INJECT_FAILED
140186 	return -1;
141187 cmpfail:
188
+	rseq_after_asm_goto();
142189 	return 1;
143190 #ifdef RSEQ_COMPARE_TWICE
144191 error1:
192
+	rseq_after_asm_goto();
145193 	rseq_bug("cpu_id comparison failed");
146194 error2:
195
+	rseq_after_asm_goto();
147196 	rseq_bug("expected value comparison failed");
148197 #endif
149198 }
....@@ -154,12 +203,17 @@
154203  */
155204 static inline __attribute__((always_inline))
156205 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
157
-			       off_t voffp, intptr_t *load, int cpu)
206
+			       long voffp, intptr_t *load, int cpu)
158207 {
159208 	RSEQ_INJECT_C(9)
160209 
161210 	__asm__ __volatile__ goto (
162211 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
212
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
213
+#ifdef RSEQ_COMPARE_TWICE
214
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
215
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
216
+#endif
163217 		/* Start rseq by storing table entry pointer into rseq_cs. */
164218 		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
165219 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
....@@ -184,8 +238,8 @@
184238 		RSEQ_ASM_DEFINE_ABORT(4, "", abort)
185239 		: /* gcc asm goto does not allow outputs */
186240 		: [cpu_id]		"r" (cpu),
187
-		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
188
-		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
241
+		  [current_cpu_id]	"m" (rseq_get_abi()->cpu_id),
242
+		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
189243 		  /* final store input */
190244 		  [v]			"m" (*v),
191245 		  [expectnot]		"r" (expectnot),
....@@ -199,16 +253,21 @@
199253 		  , error1, error2
200254 #endif
201255 	);
256
+	rseq_after_asm_goto();
202257 	return 0;
203258 abort:
259
+	rseq_after_asm_goto();
204260 	RSEQ_INJECT_FAILED
205261 	return -1;
206262 cmpfail:
263
+	rseq_after_asm_goto();
207264 	return 1;
208265 #ifdef RSEQ_COMPARE_TWICE
209266 error1:
267
+	rseq_after_asm_goto();
210268 	rseq_bug("cpu_id comparison failed");
211269 error2:
270
+	rseq_after_asm_goto();
212271 	rseq_bug("expected value comparison failed");
213272 #endif
214273 }
....@@ -220,6 +279,9 @@
220279 
221280 	__asm__ __volatile__ goto (
222281 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
282
+#ifdef RSEQ_COMPARE_TWICE
283
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
284
+#endif
223285 		/* Start rseq by storing table entry pointer into rseq_cs. */
224286 		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
225287 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
....@@ -236,8 +298,8 @@
236298 		RSEQ_ASM_DEFINE_ABORT(4, "", abort)
237299 		: /* gcc asm goto does not allow outputs */
238300 		: [cpu_id]		"r" (cpu),
239
-		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
240
-		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
301
+		  [current_cpu_id]	"m" (rseq_get_abi()->cpu_id),
302
+		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
241303 		  /* final store input */
242304 		  [v]			"m" (*v),
243305 		  [count]		"r" (count)
....@@ -249,12 +311,15 @@
249311 		  , error1
250312 #endif
251313 	);
314
+	rseq_after_asm_goto();
252315 	return 0;
253316 abort:
317
+	rseq_after_asm_goto();
254318 	RSEQ_INJECT_FAILED
255319 	return -1;
256320 #ifdef RSEQ_COMPARE_TWICE
257321 error1:
322
+	rseq_after_asm_goto();
258323 	rseq_bug("cpu_id comparison failed");
259324 #endif
260325 }
....@@ -268,6 +333,11 @@
268333 
269334 	__asm__ __volatile__ goto (
270335 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
336
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
337
+#ifdef RSEQ_COMPARE_TWICE
338
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
339
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
340
+#endif
271341 		/* Start rseq by storing table entry pointer into rseq_cs. */
272342 		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
273343 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
....@@ -290,8 +360,8 @@
290360 		RSEQ_ASM_DEFINE_ABORT(4, "", abort)
291361 		: /* gcc asm goto does not allow outputs */
292362 		: [cpu_id]		"r" (cpu),
293
-		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
294
-		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
363
+		  [current_cpu_id]	"m" (rseq_get_abi()->cpu_id),
364
+		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
295365 		  /* try store input */
296366 		  [v2]			"m" (*v2),
297367 		  [newv2]		"r" (newv2),
....@@ -307,16 +377,21 @@
307377 		  , error1, error2
308378 #endif
309379 	);
380
+	rseq_after_asm_goto();
310381 	return 0;
311382 abort:
383
+	rseq_after_asm_goto();
312384 	RSEQ_INJECT_FAILED
313385 	return -1;
314386 cmpfail:
387
+	rseq_after_asm_goto();
315388 	return 1;
316389 #ifdef RSEQ_COMPARE_TWICE
317390 error1:
391
+	rseq_after_asm_goto();
318392 	rseq_bug("cpu_id comparison failed");
319393 error2:
394
+	rseq_after_asm_goto();
320395 	rseq_bug("expected value comparison failed");
321396 #endif
322397 }
....@@ -339,6 +414,12 @@
339414 
340415 	__asm__ __volatile__ goto (
341416 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
417
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
418
+#ifdef RSEQ_COMPARE_TWICE
419
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
420
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
421
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
422
+#endif
342423 		/* Start rseq by storing table entry pointer into rseq_cs. */
343424 		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
344425 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
....@@ -363,8 +444,8 @@
363444 		RSEQ_ASM_DEFINE_ABORT(4, "", abort)
364445 		: /* gcc asm goto does not allow outputs */
365446 		: [cpu_id]		"r" (cpu),
366
-		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
367
-		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
447
+		  [current_cpu_id]	"m" (rseq_get_abi()->cpu_id),
448
+		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
368449 		  /* cmp2 input */
369450 		  [v2]			"m" (*v2),
370451 		  [expect2]		"r" (expect2),
....@@ -380,18 +461,24 @@
380461 		  , error1, error2, error3
381462 #endif
382463 	);
464
+	rseq_after_asm_goto();
383465 	return 0;
384466 abort:
467
+	rseq_after_asm_goto();
385468 	RSEQ_INJECT_FAILED
386469 	return -1;
387470 cmpfail:
471
+	rseq_after_asm_goto();
388472 	return 1;
389473 #ifdef RSEQ_COMPARE_TWICE
390474 error1:
475
+	rseq_after_asm_goto();
391476 	rseq_bug("cpu_id comparison failed");
392477 error2:
478
+	rseq_after_asm_goto();
393479 	rseq_bug("1st expected value comparison failed");
394480 error3:
481
+	rseq_after_asm_goto();
395482 	rseq_bug("2nd expected value comparison failed");
396483 #endif
397484 }
....@@ -407,6 +494,11 @@
407494 
408495 	__asm__ __volatile__ goto (
409496 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
497
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
498
+#ifdef RSEQ_COMPARE_TWICE
499
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
500
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
501
+#endif
410502 		LONG_S " %[src], %[rseq_scratch0]\n\t"
411503 		LONG_S " %[dst], %[rseq_scratch1]\n\t"
412504 		LONG_S " %[len], %[rseq_scratch2]\n\t"
....@@ -466,8 +558,8 @@
466558 #endif
467559 		: /* gcc asm goto does not allow outputs */
468560 		: [cpu_id]		"r" (cpu),
469
-		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
470
-		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
561
+		  [current_cpu_id]	"m" (rseq_get_abi()->cpu_id),
562
+		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
471563 		  /* final store input */
472564 		  [v]			"m" (*v),
473565 		  [expect]		"r" (expect),
....@@ -487,16 +579,21 @@
487579 		  , error1, error2
488580 #endif
489581 	);
582
+	rseq_after_asm_goto();
490583 	return 0;
491584 abort:
585
+	rseq_after_asm_goto();
492586 	RSEQ_INJECT_FAILED
493587 	return -1;
494588 cmpfail:
589
+	rseq_after_asm_goto();
495590 	return 1;
496591 #ifdef RSEQ_COMPARE_TWICE
497592 error1:
593
+	rseq_after_asm_goto();
498594 	rseq_bug("cpu_id comparison failed");
499595 error2:
596
+	rseq_after_asm_goto();
500597 	rseq_bug("expected value comparison failed");
501598 #endif
502599 }