blame | history | patch | commit | commitdiff
hc
2023-12-08 01573e231f18eb2d99162747186f59511f56b64d 
 kernel/tools/testing/selftests/rseq/rseq-ppc.h
....@@ -6,7 +6,15 @@
66  * (C) Copyright 2016-2018 - Boqun Feng <boqun.feng@gmail.com>
77  */
88 
9
-#define RSEQ_SIG	0x53053053
9
+/*
10
+ * RSEQ_SIG is used with the following trap instruction:
11
+ *
12
+ * powerpc-be:    0f e5 00 0b           twui   r5,11
13
+ * powerpc64-le:  0b 00 e5 0f           twui   r5,11
14
+ * powerpc64-be:  0f e5 00 0b           twui   r5,11
15
+ */
16
+
17
+#define RSEQ_SIG	0x0fe5000b
1018 
1119 #define rseq_smp_mb()		__asm__ __volatile__ ("sync"	::: "memory", "cc")
1220 #define rseq_smp_lwsync()	__asm__ __volatile__ ("lwsync"	::: "memory", "cc")
....@@ -33,24 +41,30 @@
3341 #else /* !RSEQ_SKIP_FASTPATH */
3442 
3543 /*
36
- * The __rseq_table section can be used by debuggers to better handle
37
- * single-stepping through the restartable critical sections.
44
+ * The __rseq_cs_ptr_array and __rseq_cs sections can be used by debuggers to
45
+ * better handle single-stepping through the restartable critical sections.
3846  */
3947 
4048 #ifdef __PPC64__
4149 
42
-#define STORE_WORD	"std "
43
-#define LOAD_WORD	"ld "
44
-#define LOADX_WORD	"ldx "
45
-#define CMP_WORD	"cmpd "
50
+#define RSEQ_STORE_LONG(arg)	"std%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] "	/* To memory ("m" constraint) */
51
+#define RSEQ_STORE_INT(arg)	"stw%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] "	/* To memory ("m" constraint) */
52
+#define RSEQ_LOAD_LONG(arg)	"ld%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] "	/* From memory ("m" constraint) */
53
+#define RSEQ_LOAD_INT(arg)	"lwz%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] "	/* From memory ("m" constraint) */
54
+#define RSEQ_LOADX_LONG		"ldx "							/* From base register ("b" constraint) */
55
+#define RSEQ_CMP_LONG		"cmpd "
56
+#define RSEQ_CMP_LONG_INT	"cmpdi "
4657 
4758 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,				\
4859 			start_ip, post_commit_offset, abort_ip)			\
49
-		".pushsection __rseq_table, \"aw\"\n\t"				\
60
+		".pushsection __rseq_cs, \"aw\"\n\t"				\
5061 		".balign 32\n\t"						\
5162 		__rseq_str(label) ":\n\t"					\
5263 		".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t"	\
5364 		".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
65
+		".popsection\n\t"						\
66
+		".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"			\
67
+		".quad " __rseq_str(label) "b\n\t"				\
5468 		".popsection\n\t"
5569 
5670 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)			\
....@@ -63,28 +77,61 @@
6377 		"std %%r17, %[" __rseq_str(rseq_cs) "]\n\t"			\
6478 		__rseq_str(label) ":\n\t"
6579 
80
+/*
81
+ * Exit points of a rseq critical section consist of all instructions outside
82
+ * of the critical section where a critical section can either branch to or
83
+ * reach through the normal course of its execution. The abort IP and the
84
+ * post-commit IP are already part of the __rseq_cs section and should not be
85
+ * explicitly defined as additional exit points. Knowing all exit points is
86
+ * useful to assist debuggers stepping over the critical section.
87
+ */
88
+#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)			\
89
+		".pushsection __rseq_exit_point_array, \"aw\"\n\t"	\
90
+		".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
91
+		".popsection\n\t"
92
+
6693 #else /* #ifdef __PPC64__ */
6794 
68
-#define STORE_WORD	"stw "
69
-#define LOAD_WORD	"lwz "
70
-#define LOADX_WORD	"lwzx "
71
-#define CMP_WORD	"cmpw "
95
+#define RSEQ_STORE_LONG(arg)	"stw%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] "	/* To memory ("m" constraint) */
96
+#define RSEQ_STORE_INT(arg)	RSEQ_STORE_LONG(arg)					/* To memory ("m" constraint) */
97
+#define RSEQ_LOAD_LONG(arg)	"lwz%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] "	/* From memory ("m" constraint) */
98
+#define RSEQ_LOAD_INT(arg)	RSEQ_LOAD_LONG(arg)					/* From memory ("m" constraint) */
99
+#define RSEQ_LOADX_LONG		"lwzx "							/* From base register ("b" constraint) */
100
+#define RSEQ_CMP_LONG		"cmpw "
101
+#define RSEQ_CMP_LONG_INT	"cmpwi "
72102 
73103 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,				\
74104 			start_ip, post_commit_offset, abort_ip)			\
75
-		".pushsection __rseq_table, \"aw\"\n\t"				\
105
+		".pushsection __rseq_cs, \"aw\"\n\t"				\
76106 		".balign 32\n\t"						\
77107 		__rseq_str(label) ":\n\t"					\
78108 		".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t"	\
79109 		/* 32-bit only supported on BE */				\
80110 		".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) "\n\t" \
111
+		".popsection\n\t"					\
112
+		".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"		\
113
+		".long 0x0, " __rseq_str(label) "b\n\t"			\
114
+		".popsection\n\t"
115
+
116
+/*
117
+ * Exit points of a rseq critical section consist of all instructions outside
118
+ * of the critical section where a critical section can either branch to or
119
+ * reach through the normal course of its execution. The abort IP and the
120
+ * post-commit IP are already part of the __rseq_cs section and should not be
121
+ * explicitly defined as additional exit points. Knowing all exit points is
122
+ * useful to assist debuggers stepping over the critical section.
123
+ */
124
+#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)				\
125
+		".pushsection __rseq_exit_point_array, \"aw\"\n\t"		\
126
+		/* 32-bit only supported on BE */				\
127
+		".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) "\n\t"	\
81128 		".popsection\n\t"
82129 
83130 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)			\
84131 		RSEQ_INJECT_ASM(1)						\
85132 		"lis %%r17, (" __rseq_str(cs_label) ")@ha\n\t"			\
86133 		"addi %%r17, %%r17, (" __rseq_str(cs_label) ")@l\n\t"		\
87
-		"stw %%r17, %[" __rseq_str(rseq_cs) "]\n\t"			\
134
+		RSEQ_STORE_INT(rseq_cs) "%%r17, %[" __rseq_str(rseq_cs) "]\n\t"	\
88135 		__rseq_str(label) ":\n\t"
89136 
90137 #endif /* #ifdef __PPC64__ */
....@@ -95,7 +142,7 @@
95142 
96143 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label)			\
97144 		RSEQ_INJECT_ASM(2)						\
98
-		"lwz %%r17, %[" __rseq_str(current_cpu_id) "]\n\t"		\
145
+		RSEQ_LOAD_INT(current_cpu_id) "%%r17, %[" __rseq_str(current_cpu_id) "]\n\t" \
99146 		"cmpw cr7, %[" __rseq_str(cpu_id) "], %%r17\n\t"		\
100147 		"bne- cr7, " __rseq_str(label) "\n\t"
101148 
....@@ -112,25 +159,25 @@
112159  * 	RSEQ_ASM_OP_* (else): doesn't have hard-code registers(unless cr7)
113160  */
114161 #define RSEQ_ASM_OP_CMPEQ(var, expect, label)					\
115
-		LOAD_WORD "%%r17, %[" __rseq_str(var) "]\n\t"			\
116
-		CMP_WORD "cr7, %%r17, %[" __rseq_str(expect) "]\n\t"		\
162
+		RSEQ_LOAD_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t"		\
163
+		RSEQ_CMP_LONG "cr7, %%r17, %[" __rseq_str(expect) "]\n\t"		\
117164 		"bne- cr7, " __rseq_str(label) "\n\t"
118165 
119166 #define RSEQ_ASM_OP_CMPNE(var, expectnot, label)				\
120
-		LOAD_WORD "%%r17, %[" __rseq_str(var) "]\n\t"			\
121
-		CMP_WORD "cr7, %%r17, %[" __rseq_str(expectnot) "]\n\t"		\
167
+		RSEQ_LOAD_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t"		\
168
+		RSEQ_CMP_LONG "cr7, %%r17, %[" __rseq_str(expectnot) "]\n\t"		\
122169 		"beq- cr7, " __rseq_str(label) "\n\t"
123170 
124171 #define RSEQ_ASM_OP_STORE(value, var)						\
125
-		STORE_WORD "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t"
172
+		RSEQ_STORE_LONG(var) "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t"
126173 
127174 /* Load @var to r17 */
128175 #define RSEQ_ASM_OP_R_LOAD(var)							\
129
-		LOAD_WORD "%%r17, %[" __rseq_str(var) "]\n\t"
176
+		RSEQ_LOAD_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t"
130177 
131178 /* Store r17 to @var */
132179 #define RSEQ_ASM_OP_R_STORE(var)						\
133
-		STORE_WORD "%%r17, %[" __rseq_str(var) "]\n\t"
180
+		RSEQ_STORE_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t"
134181 
135182 /* Add @count to r17 */
136183 #define RSEQ_ASM_OP_R_ADD(count)						\
....@@ -138,11 +185,11 @@
138185 
139186 /* Load (r17 + voffp) to r17 */
140187 #define RSEQ_ASM_OP_R_LOADX(voffp)						\
141
-		LOADX_WORD "%%r17, %[" __rseq_str(voffp) "], %%r17\n\t"
188
+		RSEQ_LOADX_LONG "%%r17, %[" __rseq_str(voffp) "], %%r17\n\t"
142189 
143190 /* TODO: implement a faster memcpy. */
144191 #define RSEQ_ASM_OP_R_MEMCPY() \
145
-		"cmpdi %%r19, 0\n\t" \
192
+		RSEQ_CMP_LONG_INT "%%r19, 0\n\t" \
146193 		"beq 333f\n\t" \
147194 		"addi %%r20, %%r20, -1\n\t" \
148195 		"addi %%r21, %%r21, -1\n\t" \
....@@ -150,16 +197,16 @@
150197 		"lbzu %%r18, 1(%%r20)\n\t" \
151198 		"stbu %%r18, 1(%%r21)\n\t" \
152199 		"addi %%r19, %%r19, -1\n\t" \
153
-		"cmpdi %%r19, 0\n\t" \
200
+		RSEQ_CMP_LONG_INT "%%r19, 0\n\t" \
154201 		"bne 222b\n\t" \
155202 		"333:\n\t" \
156203 
157204 #define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label)			\
158
-		STORE_WORD "%%r17, %[" __rseq_str(var) "]\n\t"			\
205
+		RSEQ_STORE_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t"			\
159206 		__rseq_str(post_commit_label) ":\n\t"
160207 
161208 #define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label)			\
162
-		STORE_WORD "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t" \
209
+		RSEQ_STORE_LONG(var) "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t" \
163210 		__rseq_str(post_commit_label) ":\n\t"
164211 
165212 static inline __attribute__((always_inline))
....@@ -169,6 +216,11 @@
169216 
170217 	__asm__ __volatile__ goto (
171218 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
219
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
220
+#ifdef RSEQ_COMPARE_TWICE
221
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
222
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
223
+#endif
172224 		/* Start rseq by storing table entry pointer into rseq_cs. */
173225 		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
174226 		/* cmp cpuid */
....@@ -189,8 +241,8 @@
189241 		RSEQ_ASM_DEFINE_ABORT(4, abort)
190242 		: /* gcc asm goto does not allow outputs */
191243 		: [cpu_id]		"r" (cpu),
192
-		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
193
-		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
244
+		  [current_cpu_id]	"m" (rseq_get_abi()->cpu_id),
245
+		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
194246 		  [v]			"m" (*v),
195247 		  [expect]		"r" (expect),
196248 		  [newv]		"r" (newv)
....@@ -202,28 +254,38 @@
202254 		  , error1, error2
203255 #endif
204256 	);
257
+	rseq_after_asm_goto();
205258 	return 0;
206259 abort:
260
+	rseq_after_asm_goto();
207261 	RSEQ_INJECT_FAILED
208262 	return -1;
209263 cmpfail:
264
+	rseq_after_asm_goto();
210265 	return 1;
211266 #ifdef RSEQ_COMPARE_TWICE
212267 error1:
268
+	rseq_after_asm_goto();
213269 	rseq_bug("cpu_id comparison failed");
214270 error2:
271
+	rseq_after_asm_goto();
215272 	rseq_bug("expected value comparison failed");
216273 #endif
217274 }
218275 
219276 static inline __attribute__((always_inline))
220277 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
221
-			       off_t voffp, intptr_t *load, int cpu)
278
+			       long voffp, intptr_t *load, int cpu)
222279 {
223280 	RSEQ_INJECT_C(9)
224281 
225282 	__asm__ __volatile__ goto (
226283 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
284
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
285
+#ifdef RSEQ_COMPARE_TWICE
286
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
287
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
288
+#endif
227289 		/* Start rseq by storing table entry pointer into rseq_cs. */
228290 		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
229291 		/* cmp cpuid */
....@@ -250,8 +312,8 @@
250312 		RSEQ_ASM_DEFINE_ABORT(4, abort)
251313 		: /* gcc asm goto does not allow outputs */
252314 		: [cpu_id]		"r" (cpu),
253
-		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
254
-		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
315
+		  [current_cpu_id]	"m" (rseq_get_abi()->cpu_id),
316
+		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
255317 		  /* final store input */
256318 		  [v]			"m" (*v),
257319 		  [expectnot]		"r" (expectnot),
....@@ -265,16 +327,21 @@
265327 		  , error1, error2
266328 #endif
267329 	);
330
+	rseq_after_asm_goto();
268331 	return 0;
269332 abort:
333
+	rseq_after_asm_goto();
270334 	RSEQ_INJECT_FAILED
271335 	return -1;
272336 cmpfail:
337
+	rseq_after_asm_goto();
273338 	return 1;
274339 #ifdef RSEQ_COMPARE_TWICE
275340 error1:
341
+	rseq_after_asm_goto();
276342 	rseq_bug("cpu_id comparison failed");
277343 error2:
344
+	rseq_after_asm_goto();
278345 	rseq_bug("expected value comparison failed");
279346 #endif
280347 }
....@@ -286,6 +353,9 @@
286353 
287354 	__asm__ __volatile__ goto (
288355 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
356
+#ifdef RSEQ_COMPARE_TWICE
357
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
358
+#endif
289359 		/* Start rseq by storing table entry pointer into rseq_cs. */
290360 		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
291361 		/* cmp cpuid */
....@@ -305,8 +375,8 @@
305375 		RSEQ_ASM_DEFINE_ABORT(4, abort)
306376 		: /* gcc asm goto does not allow outputs */
307377 		: [cpu_id]		"r" (cpu),
308
-		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
309
-		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
378
+		  [current_cpu_id]	"m" (rseq_get_abi()->cpu_id),
379
+		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
310380 		  /* final store input */
311381 		  [v]			"m" (*v),
312382 		  [count]		"r" (count)
....@@ -318,12 +388,15 @@
318388 		  , error1
319389 #endif
320390 	);
391
+	rseq_after_asm_goto();
321392 	return 0;
322393 abort:
394
+	rseq_after_asm_goto();
323395 	RSEQ_INJECT_FAILED
324396 	return -1;
325397 #ifdef RSEQ_COMPARE_TWICE
326398 error1:
399
+	rseq_after_asm_goto();
327400 	rseq_bug("cpu_id comparison failed");
328401 #endif
329402 }
....@@ -337,6 +410,11 @@
337410 
338411 	__asm__ __volatile__ goto (
339412 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
413
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
414
+#ifdef RSEQ_COMPARE_TWICE
415
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
416
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
417
+#endif
340418 		/* Start rseq by storing table entry pointer into rseq_cs. */
341419 		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
342420 		/* cmp cpuid */
....@@ -360,8 +438,8 @@
360438 		RSEQ_ASM_DEFINE_ABORT(4, abort)
361439 		: /* gcc asm goto does not allow outputs */
362440 		: [cpu_id]		"r" (cpu),
363
-		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
364
-		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
441
+		  [current_cpu_id]	"m" (rseq_get_abi()->cpu_id),
442
+		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
365443 		  /* try store input */
366444 		  [v2]			"m" (*v2),
367445 		  [newv2]		"r" (newv2),
....@@ -377,16 +455,21 @@
377455 		  , error1, error2
378456 #endif
379457 	);
458
+	rseq_after_asm_goto();
380459 	return 0;
381460 abort:
461
+	rseq_after_asm_goto();
382462 	RSEQ_INJECT_FAILED
383463 	return -1;
384464 cmpfail:
465
+	rseq_after_asm_goto();
385466 	return 1;
386467 #ifdef RSEQ_COMPARE_TWICE
387468 error1:
469
+	rseq_after_asm_goto();
388470 	rseq_bug("cpu_id comparison failed");
389471 error2:
472
+	rseq_after_asm_goto();
390473 	rseq_bug("expected value comparison failed");
391474 #endif
392475 }
....@@ -400,6 +483,11 @@
400483 
401484 	__asm__ __volatile__ goto (
402485 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
486
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
487
+#ifdef RSEQ_COMPARE_TWICE
488
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
489
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
490
+#endif
403491 		/* Start rseq by storing table entry pointer into rseq_cs. */
404492 		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
405493 		/* cmp cpuid */
....@@ -425,8 +513,8 @@
425513 		RSEQ_ASM_DEFINE_ABORT(4, abort)
426514 		: /* gcc asm goto does not allow outputs */
427515 		: [cpu_id]		"r" (cpu),
428
-		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
429
-		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
516
+		  [current_cpu_id]	"m" (rseq_get_abi()->cpu_id),
517
+		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
430518 		  /* try store input */
431519 		  [v2]			"m" (*v2),
432520 		  [newv2]		"r" (newv2),
....@@ -442,16 +530,21 @@
442530 		  , error1, error2
443531 #endif
444532 	);
533
+	rseq_after_asm_goto();
445534 	return 0;
446535 abort:
536
+	rseq_after_asm_goto();
447537 	RSEQ_INJECT_FAILED
448538 	return -1;
449539 cmpfail:
540
+	rseq_after_asm_goto();
450541 	return 1;
451542 #ifdef RSEQ_COMPARE_TWICE
452543 error1:
544
+	rseq_after_asm_goto();
453545 	rseq_bug("cpu_id comparison failed");
454546 error2:
547
+	rseq_after_asm_goto();
455548 	rseq_bug("expected value comparison failed");
456549 #endif
457550 }
....@@ -465,6 +558,12 @@
465558 
466559 	__asm__ __volatile__ goto (
467560 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
561
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
562
+#ifdef RSEQ_COMPARE_TWICE
563
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
564
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
565
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
566
+#endif
468567 		/* Start rseq by storing table entry pointer into rseq_cs. */
469568 		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
470569 		/* cmp cpuid */
....@@ -490,8 +589,8 @@
490589 		RSEQ_ASM_DEFINE_ABORT(4, abort)
491590 		: /* gcc asm goto does not allow outputs */
492591 		: [cpu_id]		"r" (cpu),
493
-		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
494
-		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
592
+		  [current_cpu_id]	"m" (rseq_get_abi()->cpu_id),
593
+		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
495594 		  /* cmp2 input */
496595 		  [v2]			"m" (*v2),
497596 		  [expect2]		"r" (expect2),
....@@ -507,18 +606,24 @@
507606 		  , error1, error2, error3
508607 #endif
509608 	);
609
+	rseq_after_asm_goto();
510610 	return 0;
511611 abort:
612
+	rseq_after_asm_goto();
512613 	RSEQ_INJECT_FAILED
513614 	return -1;
514615 cmpfail:
616
+	rseq_after_asm_goto();
515617 	return 1;
516618 #ifdef RSEQ_COMPARE_TWICE
517619 error1:
620
+	rseq_after_asm_goto();
518621 	rseq_bug("cpu_id comparison failed");
519622 error2:
623
+	rseq_after_asm_goto();
520624 	rseq_bug("1st expected value comparison failed");
521625 error3:
626
+	rseq_after_asm_goto();
522627 	rseq_bug("2nd expected value comparison failed");
523628 #endif
524629 }
....@@ -532,6 +637,11 @@
532637 
533638 	__asm__ __volatile__ goto (
534639 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
640
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
641
+#ifdef RSEQ_COMPARE_TWICE
642
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
643
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
644
+#endif
535645 		/* setup for mempcy */
536646 		"mr %%r19, %[len]\n\t"
537647 		"mr %%r20, %[src]\n\t"
....@@ -560,8 +670,8 @@
560670 		RSEQ_ASM_DEFINE_ABORT(4, abort)
561671 		: /* gcc asm goto does not allow outputs */
562672 		: [cpu_id]		"r" (cpu),
563
-		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
564
-		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
673
+		  [current_cpu_id]	"m" (rseq_get_abi()->cpu_id),
674
+		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
565675 		  /* final store input */
566676 		  [v]			"m" (*v),
567677 		  [expect]		"r" (expect),
....@@ -578,16 +688,21 @@
578688 		  , error1, error2
579689 #endif
580690 	);
691
+	rseq_after_asm_goto();
581692 	return 0;
582693 abort:
694
+	rseq_after_asm_goto();
583695 	RSEQ_INJECT_FAILED
584696 	return -1;
585697 cmpfail:
698
+	rseq_after_asm_goto();
586699 	return 1;
587700 #ifdef RSEQ_COMPARE_TWICE
588701 error1:
702
+	rseq_after_asm_goto();
589703 	rseq_bug("cpu_id comparison failed");
590704 error2:
705
+	rseq_after_asm_goto();
591706 	rseq_bug("expected value comparison failed");
592707 #endif
593708 }
....@@ -601,6 +716,11 @@
601716 
602717 	__asm__ __volatile__ goto (
603718 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
719
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
720
+#ifdef RSEQ_COMPARE_TWICE
721
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
722
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
723
+#endif
604724 		/* setup for mempcy */
605725 		"mr %%r19, %[len]\n\t"
606726 		"mr %%r20, %[src]\n\t"
....@@ -631,8 +751,8 @@
631751 		RSEQ_ASM_DEFINE_ABORT(4, abort)
632752 		: /* gcc asm goto does not allow outputs */
633753 		: [cpu_id]		"r" (cpu),
634
-		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
635
-		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
754
+		  [current_cpu_id]	"m" (rseq_get_abi()->cpu_id),
755
+		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
636756 		  /* final store input */
637757 		  [v]			"m" (*v),
638758 		  [expect]		"r" (expect),
....@@ -649,23 +769,23 @@
649769 		  , error1, error2
650770 #endif
651771 	);
772
+	rseq_after_asm_goto();
652773 	return 0;
653774 abort:
775
+	rseq_after_asm_goto();
654776 	RSEQ_INJECT_FAILED
655777 	return -1;
656778 cmpfail:
779
+	rseq_after_asm_goto();
657780 	return 1;
658781 #ifdef RSEQ_COMPARE_TWICE
659782 error1:
783
+	rseq_after_asm_goto();
660784 	rseq_bug("cpu_id comparison failed");
661785 error2:
786
+	rseq_after_asm_goto();
662787 	rseq_bug("expected value comparison failed");
663788 #endif
664789 }
665
-
666
-#undef STORE_WORD
667
-#undef LOAD_WORD
668
-#undef LOADX_WORD
669
-#undef CMP_WORD
670790 
671791 #endif /* !RSEQ_SKIP_FASTPATH */