forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2024-02-20 102a0743326a03cd1a1202ceda21e175b7d3575c 
 kernel/tools/testing/selftests/rseq/rseq-x86.h
....@@ -7,9 +7,28 @@
77 
88 #include <stdint.h>
99 
10
+/*
11
+ * RSEQ_SIG is used with the following reserved undefined instructions, which
12
+ * trap in user-space:
13
+ *
14
+ * x86-32:    0f b9 3d 53 30 05 53      ud1    0x53053053,%edi
15
+ * x86-64:    0f b9 3d 53 30 05 53      ud1    0x53053053(%rip),%edi
16
+ */
1017 #define RSEQ_SIG	0x53053053
1118 
19
+/*
20
+ * Due to a compiler optimization bug in gcc-8 with asm goto and TLS asm input
21
+ * operands, we cannot use "m" input operands, and rather pass the __rseq_abi
22
+ * address through a "r" input operand.
23
+ */
24
+
25
+/* Offset of cpu_id and rseq_cs fields in struct rseq. */
26
+#define RSEQ_CPU_ID_OFFSET	4
27
+#define RSEQ_CS_OFFSET		8
28
+
1229 #ifdef __x86_64__
30
+
31
+#define RSEQ_ASM_TP_SEGMENT	%%fs
1332 
1433 #define rseq_smp_mb()	\
1534 	__asm__ __volatile__ ("lock; addl $0,-128(%%rsp)" ::: "memory", "cc")
....@@ -37,32 +56,49 @@
3756 
3857 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,			\
3958 				start_ip, post_commit_offset, abort_ip)	\
40
-		".pushsection __rseq_table, \"aw\"\n\t"			\
59
+		".pushsection __rseq_cs, \"aw\"\n\t"			\
4160 		".balign 32\n\t"					\
4261 		__rseq_str(label) ":\n\t"				\
4362 		".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
4463 		".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
64
+		".popsection\n\t"					\
65
+		".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"		\
66
+		".quad " __rseq_str(label) "b\n\t"			\
4567 		".popsection\n\t"
68
+
4669 
4770 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
4871 	__RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,		\
4972 				(post_commit_ip - start_ip), abort_ip)
5073 
74
+/*
75
+ * Exit points of a rseq critical section consist of all instructions outside
76
+ * of the critical section where a critical section can either branch to or
77
+ * reach through the normal course of its execution. The abort IP and the
78
+ * post-commit IP are already part of the __rseq_cs section and should not be
79
+ * explicitly defined as additional exit points. Knowing all exit points is
80
+ * useful to assist debuggers stepping over the critical section.
81
+ */
82
+#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)			\
83
+		".pushsection __rseq_exit_point_array, \"aw\"\n\t"	\
84
+		".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
85
+		".popsection\n\t"
86
+
5187 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)		\
5288 		RSEQ_INJECT_ASM(1)					\
5389 		"leaq " __rseq_str(cs_label) "(%%rip), %%rax\n\t"	\
54
-		"movq %%rax, %[" __rseq_str(rseq_cs) "]\n\t"		\
90
+		"movq %%rax, " __rseq_str(rseq_cs) "\n\t"		\
5591 		__rseq_str(label) ":\n\t"
5692 
5793 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label)		\
5894 		RSEQ_INJECT_ASM(2)					\
59
-		"cmpl %[" __rseq_str(cpu_id) "], %[" __rseq_str(current_cpu_id) "]\n\t" \
95
+		"cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \
6096 		"jnz " __rseq_str(label) "\n\t"
6197 
6298 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label)		\
6399 		".pushsection __rseq_failure, \"ax\"\n\t"		\
64
-		/* Disassembler-friendly signature: nopl <sig>(%rip). */\
65
-		".byte 0x0f, 0x1f, 0x05\n\t"				\
100
+		/* Disassembler-friendly signature: ud1 <sig>(%rip),%edi. */ \
101
+		".byte 0x0f, 0xb9, 0x3d\n\t"				\
66102 		".long " __rseq_str(RSEQ_SIG) "\n\t"			\
67103 		__rseq_str(label) ":\n\t"				\
68104 		teardown						\
....@@ -83,15 +119,20 @@
83119 
84120 	__asm__ __volatile__ goto (
85121 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
122
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
123
+#ifdef RSEQ_COMPARE_TWICE
124
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
125
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
126
+#endif
86127 		/* Start rseq by storing table entry pointer into rseq_cs. */
87
-		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
88
-		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
128
+		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
129
+		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
89130 		RSEQ_INJECT_ASM(3)
90131 		"cmpq %[v], %[expect]\n\t"
91132 		"jnz %l[cmpfail]\n\t"
92133 		RSEQ_INJECT_ASM(4)
93134 #ifdef RSEQ_COMPARE_TWICE
94
-		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
135
+		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
95136 		"cmpq %[v], %[expect]\n\t"
96137 		"jnz %l[error2]\n\t"
97138 #endif
....@@ -102,8 +143,7 @@
102143 		RSEQ_ASM_DEFINE_ABORT(4, "", abort)
103144 		: /* gcc asm goto does not allow outputs */
104145 		: [cpu_id]		"r" (cpu),
105
-		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
106
-		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
146
+		  [rseq_offset]		"r" (rseq_offset),
107147 		  [v]			"m" (*v),
108148 		  [expect]		"r" (expect),
109149 		  [newv]		"r" (newv)
....@@ -114,16 +154,21 @@
114154 		  , error1, error2
115155 #endif
116156 	);
157
+	rseq_after_asm_goto();
117158 	return 0;
118159 abort:
160
+	rseq_after_asm_goto();
119161 	RSEQ_INJECT_FAILED
120162 	return -1;
121163 cmpfail:
164
+	rseq_after_asm_goto();
122165 	return 1;
123166 #ifdef RSEQ_COMPARE_TWICE
124167 error1:
168
+	rseq_after_asm_goto();
125169 	rseq_bug("cpu_id comparison failed");
126170 error2:
171
+	rseq_after_asm_goto();
127172 	rseq_bug("expected value comparison failed");
128173 #endif
129174 }
....@@ -134,22 +179,27 @@
134179  */
135180 static inline __attribute__((always_inline))
136181 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
137
-			       off_t voffp, intptr_t *load, int cpu)
182
+			       long voffp, intptr_t *load, int cpu)
138183 {
139184 	RSEQ_INJECT_C(9)
140185 
141186 	__asm__ __volatile__ goto (
142187 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
188
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
189
+#ifdef RSEQ_COMPARE_TWICE
190
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
191
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
192
+#endif
143193 		/* Start rseq by storing table entry pointer into rseq_cs. */
144
-		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
145
-		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
194
+		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
195
+		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
146196 		RSEQ_INJECT_ASM(3)
147197 		"movq %[v], %%rbx\n\t"
148198 		"cmpq %%rbx, %[expectnot]\n\t"
149199 		"je %l[cmpfail]\n\t"
150200 		RSEQ_INJECT_ASM(4)
151201 #ifdef RSEQ_COMPARE_TWICE
152
-		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
202
+		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
153203 		"movq %[v], %%rbx\n\t"
154204 		"cmpq %%rbx, %[expectnot]\n\t"
155205 		"je %l[error2]\n\t"
....@@ -164,8 +214,7 @@
164214 		RSEQ_ASM_DEFINE_ABORT(4, "", abort)
165215 		: /* gcc asm goto does not allow outputs */
166216 		: [cpu_id]		"r" (cpu),
167
-		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
168
-		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
217
+		  [rseq_offset]		"r" (rseq_offset),
169218 		  /* final store input */
170219 		  [v]			"m" (*v),
171220 		  [expectnot]		"r" (expectnot),
....@@ -178,16 +227,21 @@
178227 		  , error1, error2
179228 #endif
180229 	);
230
+	rseq_after_asm_goto();
181231 	return 0;
182232 abort:
233
+	rseq_after_asm_goto();
183234 	RSEQ_INJECT_FAILED
184235 	return -1;
185236 cmpfail:
237
+	rseq_after_asm_goto();
186238 	return 1;
187239 #ifdef RSEQ_COMPARE_TWICE
188240 error1:
241
+	rseq_after_asm_goto();
189242 	rseq_bug("cpu_id comparison failed");
190243 error2:
244
+	rseq_after_asm_goto();
191245 	rseq_bug("expected value comparison failed");
192246 #endif
193247 }
....@@ -199,12 +253,15 @@
199253 
200254 	__asm__ __volatile__ goto (
201255 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
256
+#ifdef RSEQ_COMPARE_TWICE
257
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
258
+#endif
202259 		/* Start rseq by storing table entry pointer into rseq_cs. */
203
-		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
204
-		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
260
+		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
261
+		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
205262 		RSEQ_INJECT_ASM(3)
206263 #ifdef RSEQ_COMPARE_TWICE
207
-		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
264
+		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
208265 #endif
209266 		/* final store */
210267 		"addq %[count], %[v]\n\t"
....@@ -213,12 +270,71 @@
213270 		RSEQ_ASM_DEFINE_ABORT(4, "", abort)
214271 		: /* gcc asm goto does not allow outputs */
215272 		: [cpu_id]		"r" (cpu),
216
-		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
217
-		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
273
+		  [rseq_offset]		"r" (rseq_offset),
218274 		  /* final store input */
219275 		  [v]			"m" (*v),
220276 		  [count]		"er" (count)
221277 		: "memory", "cc", "rax"
278
+		  RSEQ_INJECT_CLOBBER
279
+		: abort
280
+#ifdef RSEQ_COMPARE_TWICE
281
+		  , error1
282
+#endif
283
+	);
284
+	rseq_after_asm_goto();
285
+	return 0;
286
+abort:
287
+	rseq_after_asm_goto();
288
+	RSEQ_INJECT_FAILED
289
+	return -1;
290
+#ifdef RSEQ_COMPARE_TWICE
291
+error1:
292
+	rseq_after_asm_goto();
293
+	rseq_bug("cpu_id comparison failed");
294
+#endif
295
+}
296
+
297
+#define RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV
298
+
299
+/*
300
+ *   pval = *(ptr+off)
301
+ *  *pval += inc;
302
+ */
303
+static inline __attribute__((always_inline))
304
+int rseq_offset_deref_addv(intptr_t *ptr, long off, intptr_t inc, int cpu)
305
+{
306
+	RSEQ_INJECT_C(9)
307
+
308
+	__asm__ __volatile__ goto (
309
+		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
310
+#ifdef RSEQ_COMPARE_TWICE
311
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
312
+#endif
313
+		/* Start rseq by storing table entry pointer into rseq_cs. */
314
+		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
315
+		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
316
+		RSEQ_INJECT_ASM(3)
317
+#ifdef RSEQ_COMPARE_TWICE
318
+		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
319
+#endif
320
+		/* get p+v */
321
+		"movq %[ptr], %%rbx\n\t"
322
+		"addq %[off], %%rbx\n\t"
323
+		/* get pv */
324
+		"movq (%%rbx), %%rcx\n\t"
325
+		/* *pv += inc */
326
+		"addq %[inc], (%%rcx)\n\t"
327
+		"2:\n\t"
328
+		RSEQ_INJECT_ASM(4)
329
+		RSEQ_ASM_DEFINE_ABORT(4, "", abort)
330
+		: /* gcc asm goto does not allow outputs */
331
+		: [cpu_id]		"r" (cpu),
332
+		  [rseq_offset]		"r" (rseq_offset),
333
+		  /* final store input */
334
+		  [ptr]			"m" (*ptr),
335
+		  [off]			"er" (off),
336
+		  [inc]			"er" (inc)
337
+		: "memory", "cc", "rax", "rbx", "rcx"
222338 		  RSEQ_INJECT_CLOBBER
223339 		: abort
224340 #ifdef RSEQ_COMPARE_TWICE
....@@ -244,15 +360,20 @@
244360 
245361 	__asm__ __volatile__ goto (
246362 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
363
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
364
+#ifdef RSEQ_COMPARE_TWICE
365
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
366
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
367
+#endif
247368 		/* Start rseq by storing table entry pointer into rseq_cs. */
248
-		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
249
-		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
369
+		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
370
+		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
250371 		RSEQ_INJECT_ASM(3)
251372 		"cmpq %[v], %[expect]\n\t"
252373 		"jnz %l[cmpfail]\n\t"
253374 		RSEQ_INJECT_ASM(4)
254375 #ifdef RSEQ_COMPARE_TWICE
255
-		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
376
+		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
256377 		"cmpq %[v], %[expect]\n\t"
257378 		"jnz %l[error2]\n\t"
258379 #endif
....@@ -266,8 +387,7 @@
266387 		RSEQ_ASM_DEFINE_ABORT(4, "", abort)
267388 		: /* gcc asm goto does not allow outputs */
268389 		: [cpu_id]		"r" (cpu),
269
-		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
270
-		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
390
+		  [rseq_offset]		"r" (rseq_offset),
271391 		  /* try store input */
272392 		  [v2]			"m" (*v2),
273393 		  [newv2]		"r" (newv2),
....@@ -282,16 +402,21 @@
282402 		  , error1, error2
283403 #endif
284404 	);
405
+	rseq_after_asm_goto();
285406 	return 0;
286407 abort:
408
+	rseq_after_asm_goto();
287409 	RSEQ_INJECT_FAILED
288410 	return -1;
289411 cmpfail:
412
+	rseq_after_asm_goto();
290413 	return 1;
291414 #ifdef RSEQ_COMPARE_TWICE
292415 error1:
416
+	rseq_after_asm_goto();
293417 	rseq_bug("cpu_id comparison failed");
294418 error2:
419
+	rseq_after_asm_goto();
295420 	rseq_bug("expected value comparison failed");
296421 #endif
297422 }
....@@ -314,9 +439,15 @@
314439 
315440 	__asm__ __volatile__ goto (
316441 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
442
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
443
+#ifdef RSEQ_COMPARE_TWICE
444
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
445
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
446
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
447
+#endif
317448 		/* Start rseq by storing table entry pointer into rseq_cs. */
318
-		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
319
-		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
449
+		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
450
+		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
320451 		RSEQ_INJECT_ASM(3)
321452 		"cmpq %[v], %[expect]\n\t"
322453 		"jnz %l[cmpfail]\n\t"
....@@ -325,7 +456,7 @@
325456 		"jnz %l[cmpfail]\n\t"
326457 		RSEQ_INJECT_ASM(5)
327458 #ifdef RSEQ_COMPARE_TWICE
328
-		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
459
+		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
329460 		"cmpq %[v], %[expect]\n\t"
330461 		"jnz %l[error2]\n\t"
331462 		"cmpq %[v2], %[expect2]\n\t"
....@@ -338,8 +469,7 @@
338469 		RSEQ_ASM_DEFINE_ABORT(4, "", abort)
339470 		: /* gcc asm goto does not allow outputs */
340471 		: [cpu_id]		"r" (cpu),
341
-		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
342
-		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
472
+		  [rseq_offset]		"r" (rseq_offset),
343473 		  /* cmp2 input */
344474 		  [v2]			"m" (*v2),
345475 		  [expect2]		"r" (expect2),
....@@ -354,18 +484,24 @@
354484 		  , error1, error2, error3
355485 #endif
356486 	);
487
+	rseq_after_asm_goto();
357488 	return 0;
358489 abort:
490
+	rseq_after_asm_goto();
359491 	RSEQ_INJECT_FAILED
360492 	return -1;
361493 cmpfail:
494
+	rseq_after_asm_goto();
362495 	return 1;
363496 #ifdef RSEQ_COMPARE_TWICE
364497 error1:
498
+	rseq_after_asm_goto();
365499 	rseq_bug("cpu_id comparison failed");
366500 error2:
501
+	rseq_after_asm_goto();
367502 	rseq_bug("1st expected value comparison failed");
368503 error3:
504
+	rseq_after_asm_goto();
369505 	rseq_bug("2nd expected value comparison failed");
370506 #endif
371507 }
....@@ -381,18 +517,23 @@
381517 
382518 	__asm__ __volatile__ goto (
383519 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
520
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
521
+#ifdef RSEQ_COMPARE_TWICE
522
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
523
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
524
+#endif
384525 		"movq %[src], %[rseq_scratch0]\n\t"
385526 		"movq %[dst], %[rseq_scratch1]\n\t"
386527 		"movq %[len], %[rseq_scratch2]\n\t"
387528 		/* Start rseq by storing table entry pointer into rseq_cs. */
388
-		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
389
-		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
529
+		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
530
+		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
390531 		RSEQ_INJECT_ASM(3)
391532 		"cmpq %[v], %[expect]\n\t"
392533 		"jnz 5f\n\t"
393534 		RSEQ_INJECT_ASM(4)
394535 #ifdef RSEQ_COMPARE_TWICE
395
-		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
536
+		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 6f)
396537 		"cmpq %[v], %[expect]\n\t"
397538 		"jnz 7f\n\t"
398539 #endif
....@@ -440,8 +581,7 @@
440581 #endif
441582 		: /* gcc asm goto does not allow outputs */
442583 		: [cpu_id]		"r" (cpu),
443
-		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
444
-		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
584
+		  [rseq_offset]		"r" (rseq_offset),
445585 		  /* final store input */
446586 		  [v]			"m" (*v),
447587 		  [expect]		"r" (expect),
....@@ -460,16 +600,21 @@
460600 		  , error1, error2
461601 #endif
462602 	);
603
+	rseq_after_asm_goto();
463604 	return 0;
464605 abort:
606
+	rseq_after_asm_goto();
465607 	RSEQ_INJECT_FAILED
466608 	return -1;
467609 cmpfail:
610
+	rseq_after_asm_goto();
468611 	return 1;
469612 #ifdef RSEQ_COMPARE_TWICE
470613 error1:
614
+	rseq_after_asm_goto();
471615 	rseq_bug("cpu_id comparison failed");
472616 error2:
617
+	rseq_after_asm_goto();
473618 	rseq_bug("expected value comparison failed");
474619 #endif
475620 }
....@@ -486,7 +631,9 @@
486631 
487632 #endif /* !RSEQ_SKIP_FASTPATH */
488633 
489
-#elif __i386__
634
+#elif defined(__i386__)
635
+
636
+#define RSEQ_ASM_TP_SEGMENT	%%gs
490637 
491638 #define rseq_smp_mb()	\
492639 	__asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
....@@ -520,31 +667,47 @@
520667  */
521668 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,			\
522669 				start_ip, post_commit_offset, abort_ip)	\
523
-		".pushsection __rseq_table, \"aw\"\n\t"			\
670
+		".pushsection __rseq_cs, \"aw\"\n\t"			\
524671 		".balign 32\n\t"					\
525672 		__rseq_str(label) ":\n\t"				\
526673 		".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
527674 		".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
675
+		".popsection\n\t"					\
676
+		".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"		\
677
+		".long " __rseq_str(label) "b, 0x0\n\t"			\
528678 		".popsection\n\t"
529679 
530680 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
531681 	__RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,		\
532682 				(post_commit_ip - start_ip), abort_ip)
533683 
684
+/*
685
+ * Exit points of a rseq critical section consist of all instructions outside
686
+ * of the critical section where a critical section can either branch to or
687
+ * reach through the normal course of its execution. The abort IP and the
688
+ * post-commit IP are already part of the __rseq_cs section and should not be
689
+ * explicitly defined as additional exit points. Knowing all exit points is
690
+ * useful to assist debuggers stepping over the critical section.
691
+ */
692
+#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)			\
693
+		".pushsection __rseq_exit_point_array, \"aw\"\n\t"	\
694
+		".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) ", 0x0\n\t" \
695
+		".popsection\n\t"
696
+
534697 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)		\
535698 		RSEQ_INJECT_ASM(1)					\
536
-		"movl $" __rseq_str(cs_label) ", %[rseq_cs]\n\t"	\
699
+		"movl $" __rseq_str(cs_label) ", " __rseq_str(rseq_cs) "\n\t"	\
537700 		__rseq_str(label) ":\n\t"
538701 
539702 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label)		\
540703 		RSEQ_INJECT_ASM(2)					\
541
-		"cmpl %[" __rseq_str(cpu_id) "], %[" __rseq_str(current_cpu_id) "]\n\t" \
704
+		"cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \
542705 		"jnz " __rseq_str(label) "\n\t"
543706 
544707 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label)		\
545708 		".pushsection __rseq_failure, \"ax\"\n\t"		\
546
-		/* Disassembler-friendly signature: nopl <sig>. */	\
547
-		".byte 0x0f, 0x1f, 0x05\n\t"				\
709
+		/* Disassembler-friendly signature: ud1 <sig>,%edi. */	\
710
+		".byte 0x0f, 0xb9, 0x3d\n\t"				\
548711 		".long " __rseq_str(RSEQ_SIG) "\n\t"			\
549712 		__rseq_str(label) ":\n\t"				\
550713 		teardown						\
....@@ -565,15 +728,20 @@
565728 
566729 	__asm__ __volatile__ goto (
567730 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
731
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
732
+#ifdef RSEQ_COMPARE_TWICE
733
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
734
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
735
+#endif
568736 		/* Start rseq by storing table entry pointer into rseq_cs. */
569
-		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
570
-		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
737
+		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
738
+		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
571739 		RSEQ_INJECT_ASM(3)
572740 		"cmpl %[v], %[expect]\n\t"
573741 		"jnz %l[cmpfail]\n\t"
574742 		RSEQ_INJECT_ASM(4)
575743 #ifdef RSEQ_COMPARE_TWICE
576
-		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
744
+		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
577745 		"cmpl %[v], %[expect]\n\t"
578746 		"jnz %l[error2]\n\t"
579747 #endif
....@@ -584,8 +752,7 @@
584752 		RSEQ_ASM_DEFINE_ABORT(4, "", abort)
585753 		: /* gcc asm goto does not allow outputs */
586754 		: [cpu_id]		"r" (cpu),
587
-		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
588
-		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
755
+		  [rseq_offset]		"r" (rseq_offset),
589756 		  [v]			"m" (*v),
590757 		  [expect]		"r" (expect),
591758 		  [newv]		"r" (newv)
....@@ -596,16 +763,21 @@
596763 		  , error1, error2
597764 #endif
598765 	);
766
+	rseq_after_asm_goto();
599767 	return 0;
600768 abort:
769
+	rseq_after_asm_goto();
601770 	RSEQ_INJECT_FAILED
602771 	return -1;
603772 cmpfail:
773
+	rseq_after_asm_goto();
604774 	return 1;
605775 #ifdef RSEQ_COMPARE_TWICE
606776 error1:
777
+	rseq_after_asm_goto();
607778 	rseq_bug("cpu_id comparison failed");
608779 error2:
780
+	rseq_after_asm_goto();
609781 	rseq_bug("expected value comparison failed");
610782 #endif
611783 }
....@@ -616,22 +788,27 @@
616788  */
617789 static inline __attribute__((always_inline))
618790 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
619
-			       off_t voffp, intptr_t *load, int cpu)
791
+			       long voffp, intptr_t *load, int cpu)
620792 {
621793 	RSEQ_INJECT_C(9)
622794 
623795 	__asm__ __volatile__ goto (
624796 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
797
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
798
+#ifdef RSEQ_COMPARE_TWICE
799
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
800
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
801
+#endif
625802 		/* Start rseq by storing table entry pointer into rseq_cs. */
626
-		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
627
-		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
803
+		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
804
+		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
628805 		RSEQ_INJECT_ASM(3)
629806 		"movl %[v], %%ebx\n\t"
630807 		"cmpl %%ebx, %[expectnot]\n\t"
631808 		"je %l[cmpfail]\n\t"
632809 		RSEQ_INJECT_ASM(4)
633810 #ifdef RSEQ_COMPARE_TWICE
634
-		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
811
+		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
635812 		"movl %[v], %%ebx\n\t"
636813 		"cmpl %%ebx, %[expectnot]\n\t"
637814 		"je %l[error2]\n\t"
....@@ -646,8 +823,7 @@
646823 		RSEQ_ASM_DEFINE_ABORT(4, "", abort)
647824 		: /* gcc asm goto does not allow outputs */
648825 		: [cpu_id]		"r" (cpu),
649
-		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
650
-		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
826
+		  [rseq_offset]		"r" (rseq_offset),
651827 		  /* final store input */
652828 		  [v]			"m" (*v),
653829 		  [expectnot]		"r" (expectnot),
....@@ -660,16 +836,21 @@
660836 		  , error1, error2
661837 #endif
662838 	);
839
+	rseq_after_asm_goto();
663840 	return 0;
664841 abort:
842
+	rseq_after_asm_goto();
665843 	RSEQ_INJECT_FAILED
666844 	return -1;
667845 cmpfail:
846
+	rseq_after_asm_goto();
668847 	return 1;
669848 #ifdef RSEQ_COMPARE_TWICE
670849 error1:
850
+	rseq_after_asm_goto();
671851 	rseq_bug("cpu_id comparison failed");
672852 error2:
853
+	rseq_after_asm_goto();
673854 	rseq_bug("expected value comparison failed");
674855 #endif
675856 }
....@@ -681,12 +862,15 @@
681862 
682863 	__asm__ __volatile__ goto (
683864 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
865
+#ifdef RSEQ_COMPARE_TWICE
866
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
867
+#endif
684868 		/* Start rseq by storing table entry pointer into rseq_cs. */
685
-		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
686
-		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
869
+		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
870
+		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
687871 		RSEQ_INJECT_ASM(3)
688872 #ifdef RSEQ_COMPARE_TWICE
689
-		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
873
+		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
690874 #endif
691875 		/* final store */
692876 		"addl %[count], %[v]\n\t"
....@@ -695,8 +879,7 @@
695879 		RSEQ_ASM_DEFINE_ABORT(4, "", abort)
696880 		: /* gcc asm goto does not allow outputs */
697881 		: [cpu_id]		"r" (cpu),
698
-		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
699
-		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
882
+		  [rseq_offset]		"r" (rseq_offset),
700883 		  /* final store input */
701884 		  [v]			"m" (*v),
702885 		  [count]		"ir" (count)
....@@ -707,12 +890,15 @@
707890 		  , error1
708891 #endif
709892 	);
893
+	rseq_after_asm_goto();
710894 	return 0;
711895 abort:
896
+	rseq_after_asm_goto();
712897 	RSEQ_INJECT_FAILED
713898 	return -1;
714899 #ifdef RSEQ_COMPARE_TWICE
715900 error1:
901
+	rseq_after_asm_goto();
716902 	rseq_bug("cpu_id comparison failed");
717903 #endif
718904 }
....@@ -726,15 +912,20 @@
726912 
727913 	__asm__ __volatile__ goto (
728914 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
915
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
916
+#ifdef RSEQ_COMPARE_TWICE
917
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
918
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
919
+#endif
729920 		/* Start rseq by storing table entry pointer into rseq_cs. */
730
-		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
731
-		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
921
+		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
922
+		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
732923 		RSEQ_INJECT_ASM(3)
733924 		"cmpl %[v], %[expect]\n\t"
734925 		"jnz %l[cmpfail]\n\t"
735926 		RSEQ_INJECT_ASM(4)
736927 #ifdef RSEQ_COMPARE_TWICE
737
-		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
928
+		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
738929 		"cmpl %[v], %[expect]\n\t"
739930 		"jnz %l[error2]\n\t"
740931 #endif
....@@ -749,8 +940,7 @@
749940 		RSEQ_ASM_DEFINE_ABORT(4, "", abort)
750941 		: /* gcc asm goto does not allow outputs */
751942 		: [cpu_id]		"r" (cpu),
752
-		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
753
-		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
943
+		  [rseq_offset]		"r" (rseq_offset),
754944 		  /* try store input */
755945 		  [v2]			"m" (*v2),
756946 		  [newv2]		"m" (newv2),
....@@ -765,16 +955,21 @@
765955 		  , error1, error2
766956 #endif
767957 	);
958
+	rseq_after_asm_goto();
768959 	return 0;
769960 abort:
961
+	rseq_after_asm_goto();
770962 	RSEQ_INJECT_FAILED
771963 	return -1;
772964 cmpfail:
965
+	rseq_after_asm_goto();
773966 	return 1;
774967 #ifdef RSEQ_COMPARE_TWICE
775968 error1:
969
+	rseq_after_asm_goto();
776970 	rseq_bug("cpu_id comparison failed");
777971 error2:
972
+	rseq_after_asm_goto();
778973 	rseq_bug("expected value comparison failed");
779974 #endif
780975 }
....@@ -788,16 +983,21 @@
788983 
789984 	__asm__ __volatile__ goto (
790985 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
986
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
987
+#ifdef RSEQ_COMPARE_TWICE
988
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
989
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
990
+#endif
791991 		/* Start rseq by storing table entry pointer into rseq_cs. */
792
-		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
793
-		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
992
+		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
993
+		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
794994 		RSEQ_INJECT_ASM(3)
795995 		"movl %[expect], %%eax\n\t"
796996 		"cmpl %[v], %%eax\n\t"
797997 		"jnz %l[cmpfail]\n\t"
798998 		RSEQ_INJECT_ASM(4)
799999 #ifdef RSEQ_COMPARE_TWICE
800
-		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
1000
+		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
8011001 		"movl %[expect], %%eax\n\t"
8021002 		"cmpl %[v], %%eax\n\t"
8031003 		"jnz %l[error2]\n\t"
....@@ -813,8 +1013,7 @@
8131013 		RSEQ_ASM_DEFINE_ABORT(4, "", abort)
8141014 		: /* gcc asm goto does not allow outputs */
8151015 		: [cpu_id]		"r" (cpu),
816
-		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
817
-		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
1016
+		  [rseq_offset]		"r" (rseq_offset),
8181017 		  /* try store input */
8191018 		  [v2]			"m" (*v2),
8201019 		  [newv2]		"r" (newv2),
....@@ -829,16 +1028,21 @@
8291028 		  , error1, error2
8301029 #endif
8311030 	);
1031
+	rseq_after_asm_goto();
8321032 	return 0;
8331033 abort:
1034
+	rseq_after_asm_goto();
8341035 	RSEQ_INJECT_FAILED
8351036 	return -1;
8361037 cmpfail:
1038
+	rseq_after_asm_goto();
8371039 	return 1;
8381040 #ifdef RSEQ_COMPARE_TWICE
8391041 error1:
1042
+	rseq_after_asm_goto();
8401043 	rseq_bug("cpu_id comparison failed");
8411044 error2:
1045
+	rseq_after_asm_goto();
8421046 	rseq_bug("expected value comparison failed");
8431047 #endif
8441048 
....@@ -853,9 +1057,15 @@
8531057 
8541058 	__asm__ __volatile__ goto (
8551059 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
1060
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
1061
+#ifdef RSEQ_COMPARE_TWICE
1062
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
1063
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
1064
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
1065
+#endif
8561066 		/* Start rseq by storing table entry pointer into rseq_cs. */
857
-		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
858
-		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
1067
+		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
1068
+		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
8591069 		RSEQ_INJECT_ASM(3)
8601070 		"cmpl %[v], %[expect]\n\t"
8611071 		"jnz %l[cmpfail]\n\t"
....@@ -864,7 +1074,7 @@
8641074 		"jnz %l[cmpfail]\n\t"
8651075 		RSEQ_INJECT_ASM(5)
8661076 #ifdef RSEQ_COMPARE_TWICE
867
-		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
1077
+		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
8681078 		"cmpl %[v], %[expect]\n\t"
8691079 		"jnz %l[error2]\n\t"
8701080 		"cmpl %[expect2], %[v2]\n\t"
....@@ -878,8 +1088,7 @@
8781088 		RSEQ_ASM_DEFINE_ABORT(4, "", abort)
8791089 		: /* gcc asm goto does not allow outputs */
8801090 		: [cpu_id]		"r" (cpu),
881
-		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
882
-		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
1091
+		  [rseq_offset]		"r" (rseq_offset),
8831092 		  /* cmp2 input */
8841093 		  [v2]			"m" (*v2),
8851094 		  [expect2]		"r" (expect2),
....@@ -894,18 +1103,24 @@
8941103 		  , error1, error2, error3
8951104 #endif
8961105 	);
1106
+	rseq_after_asm_goto();
8971107 	return 0;
8981108 abort:
1109
+	rseq_after_asm_goto();
8991110 	RSEQ_INJECT_FAILED
9001111 	return -1;
9011112 cmpfail:
1113
+	rseq_after_asm_goto();
9021114 	return 1;
9031115 #ifdef RSEQ_COMPARE_TWICE
9041116 error1:
1117
+	rseq_after_asm_goto();
9051118 	rseq_bug("cpu_id comparison failed");
9061119 error2:
1120
+	rseq_after_asm_goto();
9071121 	rseq_bug("1st expected value comparison failed");
9081122 error3:
1123
+	rseq_after_asm_goto();
9091124 	rseq_bug("2nd expected value comparison failed");
9101125 #endif
9111126 }
....@@ -922,19 +1137,24 @@
9221137 
9231138 	__asm__ __volatile__ goto (
9241139 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
1140
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
1141
+#ifdef RSEQ_COMPARE_TWICE
1142
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
1143
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
1144
+#endif
9251145 		"movl %[src], %[rseq_scratch0]\n\t"
9261146 		"movl %[dst], %[rseq_scratch1]\n\t"
9271147 		"movl %[len], %[rseq_scratch2]\n\t"
9281148 		/* Start rseq by storing table entry pointer into rseq_cs. */
929
-		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
930
-		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
1149
+		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
1150
+		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
9311151 		RSEQ_INJECT_ASM(3)
9321152 		"movl %[expect], %%eax\n\t"
9331153 		"cmpl %%eax, %[v]\n\t"
9341154 		"jnz 5f\n\t"
9351155 		RSEQ_INJECT_ASM(4)
9361156 #ifdef RSEQ_COMPARE_TWICE
937
-		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
1157
+		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 6f)
9381158 		"movl %[expect], %%eax\n\t"
9391159 		"cmpl %%eax, %[v]\n\t"
9401160 		"jnz 7f\n\t"
....@@ -984,8 +1204,7 @@
9841204 #endif
9851205 		: /* gcc asm goto does not allow outputs */
9861206 		: [cpu_id]		"r" (cpu),
987
-		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
988
-		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
1207
+		  [rseq_offset]		"r" (rseq_offset),
9891208 		  /* final store input */
9901209 		  [v]			"m" (*v),
9911210 		  [expect]		"m" (expect),
....@@ -1004,16 +1223,21 @@
10041223 		  , error1, error2
10051224 #endif
10061225 	);
1226
+	rseq_after_asm_goto();
10071227 	return 0;
10081228 abort:
1229
+	rseq_after_asm_goto();
10091230 	RSEQ_INJECT_FAILED
10101231 	return -1;
10111232 cmpfail:
1233
+	rseq_after_asm_goto();
10121234 	return 1;
10131235 #ifdef RSEQ_COMPARE_TWICE
10141236 error1:
1237
+	rseq_after_asm_goto();
10151238 	rseq_bug("cpu_id comparison failed");
10161239 error2:
1240
+	rseq_after_asm_goto();
10171241 	rseq_bug("expected value comparison failed");
10181242 #endif
10191243 }
....@@ -1030,19 +1254,24 @@
10301254 
10311255 	__asm__ __volatile__ goto (
10321256 		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
1257
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
1258
+#ifdef RSEQ_COMPARE_TWICE
1259
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
1260
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
1261
+#endif
10331262 		"movl %[src], %[rseq_scratch0]\n\t"
10341263 		"movl %[dst], %[rseq_scratch1]\n\t"
10351264 		"movl %[len], %[rseq_scratch2]\n\t"
10361265 		/* Start rseq by storing table entry pointer into rseq_cs. */
1037
-		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
1038
-		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
1266
+		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
1267
+		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
10391268 		RSEQ_INJECT_ASM(3)
10401269 		"movl %[expect], %%eax\n\t"
10411270 		"cmpl %%eax, %[v]\n\t"
10421271 		"jnz 5f\n\t"
10431272 		RSEQ_INJECT_ASM(4)
10441273 #ifdef RSEQ_COMPARE_TWICE
1045
-		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
1274
+		RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 6f)
10461275 		"movl %[expect], %%eax\n\t"
10471276 		"cmpl %%eax, %[v]\n\t"
10481277 		"jnz 7f\n\t"
....@@ -1093,8 +1322,7 @@
10931322 #endif
10941323 		: /* gcc asm goto does not allow outputs */
10951324 		: [cpu_id]		"r" (cpu),
1096
-		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
1097
-		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
1325
+		  [rseq_offset]		"r" (rseq_offset),
10981326 		  /* final store input */
10991327 		  [v]			"m" (*v),
11001328 		  [expect]		"m" (expect),
....@@ -1113,16 +1341,21 @@
11131341 		  , error1, error2
11141342 #endif
11151343 	);
1344
+	rseq_after_asm_goto();
11161345 	return 0;
11171346 abort:
1347
+	rseq_after_asm_goto();
11181348 	RSEQ_INJECT_FAILED
11191349 	return -1;
11201350 cmpfail:
1351
+	rseq_after_asm_goto();
11211352 	return 1;
11221353 #ifdef RSEQ_COMPARE_TWICE
11231354 error1:
1355
+	rseq_after_asm_goto();
11241356 	rseq_bug("cpu_id comparison failed");
11251357 error2:
1358
+	rseq_after_asm_goto();
11261359 	rseq_bug("expected value comparison failed");
11271360 #endif
11281361 }