kernel_5.10 no rt

hc

2023-12-11 6778948f9de86c3cfaf36725a7c87dcff9ba247f

 kernel/tools/testing/selftests/rseq/rseq-arm.h
..
..
@@ -5,7 +5,60 @@
5
5
  * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
6
6
  */
7
7
 
8
-#define RSEQ_SIG	0x53053053
8
+/*
9
+ * - ARM little endian
10
+ *
11
+ * RSEQ_SIG uses the udf A32 instruction with an uncommon immediate operand
12
+ * value 0x5de3. This traps if user-space reaches this instruction by mistake,
13
+ * and the uncommon operand ensures the kernel does not move the instruction
14
+ * pointer to attacker-controlled code on rseq abort.
15
+ *
16
+ * The instruction pattern in the A32 instruction set is:
17
+ *
18
+ * e7f5def3    udf    #24035    ; 0x5de3
19
+ *
20
+ * This translates to the following instruction pattern in the T16 instruction
21
+ * set:
22
+ *
23
+ * little endian:
24
+ * def3        udf    #243      ; 0xf3
25
+ * e7f5        b.n    <7f5>
26
+ *
27
+ * - ARMv6+ big endian (BE8):
28
+ *
29
+ * ARMv6+ -mbig-endian generates mixed endianness code vs data: little-endian
30
+ * code and big-endian data. The data value of the signature needs to have its
31
+ * byte order reversed to generate the trap instruction:
32
+ *
33
+ * Data: 0xf3def5e7
34
+ *
35
+ * Translates to this A32 instruction pattern:
36
+ *
37
+ * e7f5def3    udf    #24035    ; 0x5de3
38
+ *
39
+ * Translates to this T16 instruction pattern:
40
+ *
41
+ * def3        udf    #243      ; 0xf3
42
+ * e7f5        b.n    <7f5>
43
+ *
44
+ * - Prior to ARMv6 big endian (BE32):
45
+ *
46
+ * Prior to ARMv6, -mbig-endian generates big-endian code and data
47
+ * (which match), so the endianness of the data representation of the
48
+ * signature should not be reversed. However, the choice between BE32
49
+ * and BE8 is done by the linker, so we cannot know whether code and
50
+ * data endianness will be mixed before the linker is invoked. So rather
51
+ * than try to play tricks with the linker, the rseq signature is simply
52
+ * data (not a trap instruction) prior to ARMv6 on big endian. This is
53
+ * why the signature is expressed as data (.word) rather than as
54
+ * instruction (.inst) in assembler.
55
+ */
56
+
57
+#ifdef __ARMEB__
58
+#define RSEQ_SIG    0xf3def5e7      /* udf    #24035    ; 0x5de3 (ARMv6+) */
59
+#else
60
+#define RSEQ_SIG    0xe7f5def3      /* udf    #24035    ; 0x5de3 */
61
+#endif
9
62
 
10
63
 #define rseq_smp_mb()	__asm__ __volatile__ ("dmb" ::: "memory", "cc")
11
64
 #define rseq_smp_rmb()	__asm__ __volatile__ ("dmb" ::: "memory", "cc")
..
..
@@ -30,17 +83,34 @@
30
83
 #include "rseq-skip.h"
31
84
 #else /* !RSEQ_SKIP_FASTPATH */
32
85
 
33
-#define __RSEQ_ASM_DEFINE_TABLE(version, flags,	start_ip,		\
86
+#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip,	\
34
87
 				post_commit_offset, abort_ip)		\
35
-		".pushsection __rseq_table, \"aw\"\n\t"			\
88
+		".pushsection __rseq_cs, \"aw\"\n\t"			\
36
89
 		".balign 32\n\t"					\
90
+		__rseq_str(label) ":\n\t"					\
37
91
 		".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
38
92
 		".word " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
93
+		".popsection\n\t"					\
94
+		".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"		\
95
+		".word " __rseq_str(label) "b, 0x0\n\t"			\
39
96
 		".popsection\n\t"
40
97
 
41
-#define RSEQ_ASM_DEFINE_TABLE(start_ip, post_commit_ip, abort_ip)	\
42
-	__RSEQ_ASM_DEFINE_TABLE(0x0, 0x0, start_ip,			\
98
+#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
99
+	__RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,		\
43
100
 				(post_commit_ip - start_ip), abort_ip)
101
+
102
+/*
103
+ * Exit points of a rseq critical section consist of all instructions outside
104
+ * of the critical section where a critical section can either branch to or
105
+ * reach through the normal course of its execution. The abort IP and the
106
+ * post-commit IP are already part of the __rseq_cs section and should not be
107
+ * explicitly defined as additional exit points. Knowing all exit points is
108
+ * useful to assist debuggers stepping over the critical section.
109
+ */
110
+#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)			\
111
+		".pushsection __rseq_exit_point_array, \"aw\"\n\t"	\
112
+		".word " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) ", 0x0\n\t" \
113
+		".popsection\n\t"
44
114
 
45
115
 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)		\
46
116
 		RSEQ_INJECT_ASM(1)					\
..
..
@@ -77,16 +147,18 @@
77
147
 		teardown						\
78
148
 		"b %l[" __rseq_str(cmpfail_label) "]\n\t"
79
149
 
80
-#define rseq_workaround_gcc_asm_size_guess()	__asm__ __volatile__("")
81
-
82
150
 static inline __attribute__((always_inline))
83
151
 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
84
152
 {
85
153
 	RSEQ_INJECT_C(9)
86
154
 
87
-	rseq_workaround_gcc_asm_size_guess();
88
155
 	__asm__ __volatile__ goto (
89
-		RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
156
+		RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
157
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
158
+#ifdef RSEQ_COMPARE_TWICE
159
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
160
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
161
+#endif
90
162
 		/* Start rseq by storing table entry pointer into rseq_cs. */
91
163
 		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
92
164
 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
..
..
@@ -110,8 +182,8 @@
110
182
 		"5:\n\t"
111
183
 		: /* gcc asm goto does not allow outputs */
112
184
 		: [cpu_id]		"r" (cpu),
113
-		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
114
-		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
185
+		  [current_cpu_id]	"m" (rseq_get_abi()->cpu_id),
186
+		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
115
187
 		  [v]			"m" (*v),
116
188
 		  [expect]		"r" (expect),
117
189
 		  [newv]		"r" (newv)
..
..
@@ -123,32 +195,38 @@
123
195
 		  , error1, error2
124
196
 #endif
125
197
 	);
126
-	rseq_workaround_gcc_asm_size_guess();
198
+	rseq_after_asm_goto();
127
199
 	return 0;
128
200
 abort:
129
-	rseq_workaround_gcc_asm_size_guess();
201
+	rseq_after_asm_goto();
130
202
 	RSEQ_INJECT_FAILED
131
203
 	return -1;
132
204
 cmpfail:
133
-	rseq_workaround_gcc_asm_size_guess();
205
+	rseq_after_asm_goto();
134
206
 	return 1;
135
207
 #ifdef RSEQ_COMPARE_TWICE
136
208
 error1:
209
+	rseq_after_asm_goto();
137
210
 	rseq_bug("cpu_id comparison failed");
138
211
 error2:
212
+	rseq_after_asm_goto();
139
213
 	rseq_bug("expected value comparison failed");
140
214
 #endif
141
215
 }
142
216
 
143
217
 static inline __attribute__((always_inline))
144
218
 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
145
-			       off_t voffp, intptr_t *load, int cpu)
219
+			       long voffp, intptr_t *load, int cpu)
146
220
 {
147
221
 	RSEQ_INJECT_C(9)
148
222
 
149
-	rseq_workaround_gcc_asm_size_guess();
150
223
 	__asm__ __volatile__ goto (
151
-		RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
224
+		RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
225
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
226
+#ifdef RSEQ_COMPARE_TWICE
227
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
228
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
229
+#endif
152
230
 		/* Start rseq by storing table entry pointer into rseq_cs. */
153
231
 		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
154
232
 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
..
..
@@ -175,8 +253,8 @@
175
253
 		"5:\n\t"
176
254
 		: /* gcc asm goto does not allow outputs */
177
255
 		: [cpu_id]		"r" (cpu),
178
-		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
179
-		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
256
+		  [current_cpu_id]	"m" (rseq_get_abi()->cpu_id),
257
+		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
180
258
 		  /* final store input */
181
259
 		  [v]			"m" (*v),
182
260
 		  [expectnot]		"r" (expectnot),
..
..
@@ -190,19 +268,21 @@
190
268
 		  , error1, error2
191
269
 #endif
192
270
 	);
193
-	rseq_workaround_gcc_asm_size_guess();
271
+	rseq_after_asm_goto();
194
272
 	return 0;
195
273
 abort:
196
-	rseq_workaround_gcc_asm_size_guess();
274
+	rseq_after_asm_goto();
197
275
 	RSEQ_INJECT_FAILED
198
276
 	return -1;
199
277
 cmpfail:
200
-	rseq_workaround_gcc_asm_size_guess();
278
+	rseq_after_asm_goto();
201
279
 	return 1;
202
280
 #ifdef RSEQ_COMPARE_TWICE
203
281
 error1:
282
+	rseq_after_asm_goto();
204
283
 	rseq_bug("cpu_id comparison failed");
205
284
 error2:
285
+	rseq_after_asm_goto();
206
286
 	rseq_bug("expected value comparison failed");
207
287
 #endif
208
288
 }
..
..
@@ -212,9 +292,11 @@
212
292
 {
213
293
 	RSEQ_INJECT_C(9)
214
294
 
215
-	rseq_workaround_gcc_asm_size_guess();
216
295
 	__asm__ __volatile__ goto (
217
-		RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
296
+		RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
297
+#ifdef RSEQ_COMPARE_TWICE
298
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
299
+#endif
218
300
 		/* Start rseq by storing table entry pointer into rseq_cs. */
219
301
 		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
220
302
 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
..
..
@@ -233,8 +315,8 @@
233
315
 		"5:\n\t"
234
316
 		: /* gcc asm goto does not allow outputs */
235
317
 		: [cpu_id]		"r" (cpu),
236
-		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
237
-		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
318
+		  [current_cpu_id]	"m" (rseq_get_abi()->cpu_id),
319
+		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
238
320
 		  [v]			"m" (*v),
239
321
 		  [count]		"Ir" (count)
240
322
 		  RSEQ_INJECT_INPUT
..
..
@@ -245,14 +327,15 @@
245
327
 		  , error1
246
328
 #endif
247
329
 	);
248
-	rseq_workaround_gcc_asm_size_guess();
330
+	rseq_after_asm_goto();
249
331
 	return 0;
250
332
 abort:
251
-	rseq_workaround_gcc_asm_size_guess();
333
+	rseq_after_asm_goto();
252
334
 	RSEQ_INJECT_FAILED
253
335
 	return -1;
254
336
 #ifdef RSEQ_COMPARE_TWICE
255
337
 error1:
338
+	rseq_after_asm_goto();
256
339
 	rseq_bug("cpu_id comparison failed");
257
340
 #endif
258
341
 }
..
..
@@ -264,9 +347,13 @@
264
347
 {
265
348
 	RSEQ_INJECT_C(9)
266
349
 
267
-	rseq_workaround_gcc_asm_size_guess();
268
350
 	__asm__ __volatile__ goto (
269
-		RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
351
+		RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
352
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
353
+#ifdef RSEQ_COMPARE_TWICE
354
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
355
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
356
+#endif
270
357
 		/* Start rseq by storing table entry pointer into rseq_cs. */
271
358
 		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
272
359
 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
..
..
@@ -293,8 +380,8 @@
293
380
 		"5:\n\t"
294
381
 		: /* gcc asm goto does not allow outputs */
295
382
 		: [cpu_id]		"r" (cpu),
296
-		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
297
-		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
383
+		  [current_cpu_id]	"m" (rseq_get_abi()->cpu_id),
384
+		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
298
385
 		  /* try store input */
299
386
 		  [v2]			"m" (*v2),
300
387
 		  [newv2]		"r" (newv2),
..
..
@@ -310,19 +397,21 @@
310
397
 		  , error1, error2
311
398
 #endif
312
399
 	);
313
-	rseq_workaround_gcc_asm_size_guess();
400
+	rseq_after_asm_goto();
314
401
 	return 0;
315
402
 abort:
316
-	rseq_workaround_gcc_asm_size_guess();
403
+	rseq_after_asm_goto();
317
404
 	RSEQ_INJECT_FAILED
318
405
 	return -1;
319
406
 cmpfail:
320
-	rseq_workaround_gcc_asm_size_guess();
407
+	rseq_after_asm_goto();
321
408
 	return 1;
322
409
 #ifdef RSEQ_COMPARE_TWICE
323
410
 error1:
411
+	rseq_after_asm_goto();
324
412
 	rseq_bug("cpu_id comparison failed");
325
413
 error2:
414
+	rseq_after_asm_goto();
326
415
 	rseq_bug("expected value comparison failed");
327
416
 #endif
328
417
 }
..
..
@@ -334,9 +423,13 @@
334
423
 {
335
424
 	RSEQ_INJECT_C(9)
336
425
 
337
-	rseq_workaround_gcc_asm_size_guess();
338
426
 	__asm__ __volatile__ goto (
339
-		RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
427
+		RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
428
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
429
+#ifdef RSEQ_COMPARE_TWICE
430
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
431
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
432
+#endif
340
433
 		/* Start rseq by storing table entry pointer into rseq_cs. */
341
434
 		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
342
435
 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
..
..
@@ -364,8 +457,8 @@
364
457
 		"5:\n\t"
365
458
 		: /* gcc asm goto does not allow outputs */
366
459
 		: [cpu_id]		"r" (cpu),
367
-		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
368
-		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
460
+		  [current_cpu_id]	"m" (rseq_get_abi()->cpu_id),
461
+		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
369
462
 		  /* try store input */
370
463
 		  [v2]			"m" (*v2),
371
464
 		  [newv2]		"r" (newv2),
..
..
@@ -381,19 +474,21 @@
381
474
 		  , error1, error2
382
475
 #endif
383
476
 	);
384
-	rseq_workaround_gcc_asm_size_guess();
477
+	rseq_after_asm_goto();
385
478
 	return 0;
386
479
 abort:
387
-	rseq_workaround_gcc_asm_size_guess();
480
+	rseq_after_asm_goto();
388
481
 	RSEQ_INJECT_FAILED
389
482
 	return -1;
390
483
 cmpfail:
391
-	rseq_workaround_gcc_asm_size_guess();
484
+	rseq_after_asm_goto();
392
485
 	return 1;
393
486
 #ifdef RSEQ_COMPARE_TWICE
394
487
 error1:
488
+	rseq_after_asm_goto();
395
489
 	rseq_bug("cpu_id comparison failed");
396
490
 error2:
491
+	rseq_after_asm_goto();
397
492
 	rseq_bug("expected value comparison failed");
398
493
 #endif
399
494
 }
..
..
@@ -405,9 +500,14 @@
405
500
 {
406
501
 	RSEQ_INJECT_C(9)
407
502
 
408
-	rseq_workaround_gcc_asm_size_guess();
409
503
 	__asm__ __volatile__ goto (
410
-		RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
504
+		RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
505
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
506
+#ifdef RSEQ_COMPARE_TWICE
507
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
508
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
509
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
510
+#endif
411
511
 		/* Start rseq by storing table entry pointer into rseq_cs. */
412
512
 		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
413
513
 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
..
..
@@ -438,8 +538,8 @@
438
538
 		"5:\n\t"
439
539
 		: /* gcc asm goto does not allow outputs */
440
540
 		: [cpu_id]		"r" (cpu),
441
-		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
442
-		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
541
+		  [current_cpu_id]	"m" (rseq_get_abi()->cpu_id),
542
+		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
443
543
 		  /* cmp2 input */
444
544
 		  [v2]			"m" (*v2),
445
545
 		  [expect2]		"r" (expect2),
..
..
@@ -455,21 +555,24 @@
455
555
 		  , error1, error2, error3
456
556
 #endif
457
557
 	);
458
-	rseq_workaround_gcc_asm_size_guess();
558
+	rseq_after_asm_goto();
459
559
 	return 0;
460
560
 abort:
461
-	rseq_workaround_gcc_asm_size_guess();
561
+	rseq_after_asm_goto();
462
562
 	RSEQ_INJECT_FAILED
463
563
 	return -1;
464
564
 cmpfail:
465
-	rseq_workaround_gcc_asm_size_guess();
565
+	rseq_after_asm_goto();
466
566
 	return 1;
467
567
 #ifdef RSEQ_COMPARE_TWICE
468
568
 error1:
569
+	rseq_after_asm_goto();
469
570
 	rseq_bug("cpu_id comparison failed");
470
571
 error2:
572
+	rseq_after_asm_goto();
471
573
 	rseq_bug("1st expected value comparison failed");
472
574
 error3:
575
+	rseq_after_asm_goto();
473
576
 	rseq_bug("2nd expected value comparison failed");
474
577
 #endif
475
578
 }
..
..
@@ -483,9 +586,13 @@
483
586
 
484
587
 	RSEQ_INJECT_C(9)
485
588
 
486
-	rseq_workaround_gcc_asm_size_guess();
487
589
 	__asm__ __volatile__ goto (
488
-		RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
590
+		RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
591
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
592
+#ifdef RSEQ_COMPARE_TWICE
593
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
594
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
595
+#endif
489
596
 		"str %[src], %[rseq_scratch0]\n\t"
490
597
 		"str %[dst], %[rseq_scratch1]\n\t"
491
598
 		"str %[len], %[rseq_scratch2]\n\t"
..
..
@@ -553,8 +660,8 @@
553
660
 		"8:\n\t"
554
661
 		: /* gcc asm goto does not allow outputs */
555
662
 		: [cpu_id]		"r" (cpu),
556
-		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
557
-		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
663
+		  [current_cpu_id]	"m" (rseq_get_abi()->cpu_id),
664
+		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
558
665
 		  /* final store input */
559
666
 		  [v]			"m" (*v),
560
667
 		  [expect]		"r" (expect),
..
..
@@ -574,21 +681,21 @@
574
681
 		  , error1, error2
575
682
 #endif
576
683
 	);
577
-	rseq_workaround_gcc_asm_size_guess();
684
+	rseq_after_asm_goto();
578
685
 	return 0;
579
686
 abort:
580
-	rseq_workaround_gcc_asm_size_guess();
687
+	rseq_after_asm_goto();
581
688
 	RSEQ_INJECT_FAILED
582
689
 	return -1;
583
690
 cmpfail:
584
-	rseq_workaround_gcc_asm_size_guess();
691
+	rseq_after_asm_goto();
585
692
 	return 1;
586
693
 #ifdef RSEQ_COMPARE_TWICE
587
694
 error1:
588
-	rseq_workaround_gcc_asm_size_guess();
695
+	rseq_after_asm_goto();
589
696
 	rseq_bug("cpu_id comparison failed");
590
697
 error2:
591
-	rseq_workaround_gcc_asm_size_guess();
698
+	rseq_after_asm_goto();
592
699
 	rseq_bug("expected value comparison failed");
593
700
 #endif
594
701
 }
..
..
@@ -602,9 +709,13 @@
602
709
 
603
710
 	RSEQ_INJECT_C(9)
604
711
 
605
-	rseq_workaround_gcc_asm_size_guess();
606
712
 	__asm__ __volatile__ goto (
607
-		RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
713
+		RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
714
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
715
+#ifdef RSEQ_COMPARE_TWICE
716
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
717
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
718
+#endif
608
719
 		"str %[src], %[rseq_scratch0]\n\t"
609
720
 		"str %[dst], %[rseq_scratch1]\n\t"
610
721
 		"str %[len], %[rseq_scratch2]\n\t"
..
..
@@ -673,8 +784,8 @@
673
784
 		"8:\n\t"
674
785
 		: /* gcc asm goto does not allow outputs */
675
786
 		: [cpu_id]		"r" (cpu),
676
-		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
677
-		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
787
+		  [current_cpu_id]	"m" (rseq_get_abi()->cpu_id),
788
+		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
678
789
 		  /* final store input */
679
790
 		  [v]			"m" (*v),
680
791
 		  [expect]		"r" (expect),
..
..
@@ -694,21 +805,21 @@
694
805
 		  , error1, error2
695
806
 #endif
696
807
 	);
697
-	rseq_workaround_gcc_asm_size_guess();
808
+	rseq_after_asm_goto();
698
809
 	return 0;
699
810
 abort:
700
-	rseq_workaround_gcc_asm_size_guess();
811
+	rseq_after_asm_goto();
701
812
 	RSEQ_INJECT_FAILED
702
813
 	return -1;
703
814
 cmpfail:
704
-	rseq_workaround_gcc_asm_size_guess();
815
+	rseq_after_asm_goto();
705
816
 	return 1;
706
817
 #ifdef RSEQ_COMPARE_TWICE
707
818
 error1:
708
-	rseq_workaround_gcc_asm_size_guess();
819
+	rseq_after_asm_goto();
709
820
 	rseq_bug("cpu_id comparison failed");
710
821
 error2:
711
-	rseq_workaround_gcc_asm_size_guess();
822
+	rseq_after_asm_goto();
712
823
 	rseq_bug("expected value comparison failed");
713
824
 #endif
714
825
 }