forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2024-02-20 102a0743326a03cd1a1202ceda21e175b7d3575c 
 kernel/tools/testing/selftests/rseq/rseq-mips.h
....@@ -7,7 +7,39 @@
77  * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
88  */
99 
10
-#define RSEQ_SIG	0x53053053
10
+/*
11
+ * RSEQ_SIG uses the break instruction. The instruction pattern is:
12
+ *
13
+ * On MIPS:
14
+ *	0350000d        break     0x350
15
+ *
16
+ * On nanoMIPS:
17
+ *      00100350        break     0x350
18
+ *
19
+ * On microMIPS:
20
+ *      0000d407        break     0x350
21
+ *
22
+ * For nanoMIPS32 and microMIPS, the instruction stream is encoded as 16-bit
23
+ * halfwords, so the signature halfwords need to be swapped accordingly for
24
+ * little-endian.
25
+ */
26
+#if defined(__nanomips__)
27
+# ifdef __MIPSEL__
28
+#  define RSEQ_SIG	0x03500010
29
+# else
30
+#  define RSEQ_SIG	0x00100350
31
+# endif
32
+#elif defined(__mips_micromips)
33
+# ifdef __MIPSEL__
34
+#  define RSEQ_SIG	0xd4070000
35
+# else
36
+#  define RSEQ_SIG	0x0000d407
37
+# endif
38
+#elif defined(__mips__)
39
+# define RSEQ_SIG	0x0350000d
40
+#else
41
+/* Unknown MIPS architecture. */
42
+#endif
1143 
1244 #define rseq_smp_mb()	__asm__ __volatile__ ("sync" ::: "memory")
1345 #define rseq_smp_rmb()	rseq_smp_mb()
....@@ -54,19 +86,37 @@
5486 # error unsupported _MIPS_SZLONG
5587 #endif
5688 
57
-#define __RSEQ_ASM_DEFINE_TABLE(version, flags,	start_ip, \
89
+#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip, \
5890 				post_commit_offset, abort_ip) \
59
-		".pushsection __rseq_table, \"aw\"\n\t" \
91
+		".pushsection __rseq_cs, \"aw\"\n\t" \
6092 		".balign 32\n\t" \
93
+		__rseq_str(label) ":\n\t"					\
6194 		".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
6295 		LONG " " U32_U64_PAD(__rseq_str(start_ip)) "\n\t" \
6396 		LONG " " U32_U64_PAD(__rseq_str(post_commit_offset)) "\n\t" \
6497 		LONG " " U32_U64_PAD(__rseq_str(abort_ip)) "\n\t" \
98
+		".popsection\n\t" \
99
+		".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
100
+		LONG " " U32_U64_PAD(__rseq_str(label) "b") "\n\t" \
65101 		".popsection\n\t"
66102 
67
-#define RSEQ_ASM_DEFINE_TABLE(start_ip, post_commit_ip, abort_ip) \
68
-	__RSEQ_ASM_DEFINE_TABLE(0x0, 0x0, start_ip, \
103
+#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
104
+	__RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
69105 				(post_commit_ip - start_ip), abort_ip)
106
+
107
+/*
108
+ * Exit points of a rseq critical section consist of all instructions outside
109
+ * of the critical section where a critical section can either branch to or
110
+ * reach through the normal course of its execution. The abort IP and the
111
+ * post-commit IP are already part of the __rseq_cs section and should not be
112
+ * explicitly defined as additional exit points. Knowing all exit points is
113
+ * useful to assist debuggers stepping over the critical section.
114
+ */
115
+#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
116
+		".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
117
+		LONG " " U32_U64_PAD(__rseq_str(start_ip)) "\n\t" \
118
+		LONG " " U32_U64_PAD(__rseq_str(exit_ip)) "\n\t" \
119
+		".popsection\n\t"
70120 
71121 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
72122 		RSEQ_INJECT_ASM(1) \
....@@ -104,16 +154,18 @@
104154 		teardown \
105155 		"b %l[" __rseq_str(cmpfail_label) "]\n\t"
106156 
107
-#define rseq_workaround_gcc_asm_size_guess()	__asm__ __volatile__("")
108
-
109157 static inline __attribute__((always_inline))
110158 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
111159 {
112160 	RSEQ_INJECT_C(9)
113161 
114
-	rseq_workaround_gcc_asm_size_guess();
115162 	__asm__ __volatile__ goto (
116
-		RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
163
+		RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
164
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
165
+#ifdef RSEQ_COMPARE_TWICE
166
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
167
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
168
+#endif
117169 		/* Start rseq by storing table entry pointer into rseq_cs. */
118170 		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
119171 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
....@@ -135,8 +187,8 @@
135187 		"5:\n\t"
136188 		: /* gcc asm goto does not allow outputs */
137189 		: [cpu_id]		"r" (cpu),
138
-		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
139
-		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
190
+		  [current_cpu_id]	"m" (rseq_get_abi()->cpu_id),
191
+		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
140192 		  [v]			"m" (*v),
141193 		  [expect]		"r" (expect),
142194 		  [newv]		"r" (newv)
....@@ -148,14 +200,11 @@
148200 		  , error1, error2
149201 #endif
150202 	);
151
-	rseq_workaround_gcc_asm_size_guess();
152203 	return 0;
153204 abort:
154
-	rseq_workaround_gcc_asm_size_guess();
155205 	RSEQ_INJECT_FAILED
156206 	return -1;
157207 cmpfail:
158
-	rseq_workaround_gcc_asm_size_guess();
159208 	return 1;
160209 #ifdef RSEQ_COMPARE_TWICE
161210 error1:
....@@ -167,13 +216,17 @@
167216 
168217 static inline __attribute__((always_inline))
169218 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
170
-			       off_t voffp, intptr_t *load, int cpu)
219
+			       long voffp, intptr_t *load, int cpu)
171220 {
172221 	RSEQ_INJECT_C(9)
173222 
174
-	rseq_workaround_gcc_asm_size_guess();
175223 	__asm__ __volatile__ goto (
176
-		RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
224
+		RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
225
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
226
+#ifdef RSEQ_COMPARE_TWICE
227
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
228
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
229
+#endif
177230 		/* Start rseq by storing table entry pointer into rseq_cs. */
178231 		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
179232 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
....@@ -198,8 +251,8 @@
198251 		"5:\n\t"
199252 		: /* gcc asm goto does not allow outputs */
200253 		: [cpu_id]		"r" (cpu),
201
-		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
202
-		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
254
+		  [current_cpu_id]	"m" (rseq_get_abi()->cpu_id),
255
+		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
203256 		  /* final store input */
204257 		  [v]			"m" (*v),
205258 		  [expectnot]		"r" (expectnot),
....@@ -213,14 +266,11 @@
213266 		  , error1, error2
214267 #endif
215268 	);
216
-	rseq_workaround_gcc_asm_size_guess();
217269 	return 0;
218270 abort:
219
-	rseq_workaround_gcc_asm_size_guess();
220271 	RSEQ_INJECT_FAILED
221272 	return -1;
222273 cmpfail:
223
-	rseq_workaround_gcc_asm_size_guess();
224274 	return 1;
225275 #ifdef RSEQ_COMPARE_TWICE
226276 error1:
....@@ -235,9 +285,11 @@
235285 {
236286 	RSEQ_INJECT_C(9)
237287 
238
-	rseq_workaround_gcc_asm_size_guess();
239288 	__asm__ __volatile__ goto (
240
-		RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
289
+		RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
290
+#ifdef RSEQ_COMPARE_TWICE
291
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
292
+#endif
241293 		/* Start rseq by storing table entry pointer into rseq_cs. */
242294 		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
243295 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
....@@ -256,8 +308,8 @@
256308 		"5:\n\t"
257309 		: /* gcc asm goto does not allow outputs */
258310 		: [cpu_id]		"r" (cpu),
259
-		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
260
-		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
311
+		  [current_cpu_id]	"m" (rseq_get_abi()->cpu_id),
312
+		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
261313 		  [v]			"m" (*v),
262314 		  [count]		"Ir" (count)
263315 		  RSEQ_INJECT_INPUT
....@@ -268,10 +320,8 @@
268320 		  , error1
269321 #endif
270322 	);
271
-	rseq_workaround_gcc_asm_size_guess();
272323 	return 0;
273324 abort:
274
-	rseq_workaround_gcc_asm_size_guess();
275325 	RSEQ_INJECT_FAILED
276326 	return -1;
277327 #ifdef RSEQ_COMPARE_TWICE
....@@ -287,9 +337,13 @@
287337 {
288338 	RSEQ_INJECT_C(9)
289339 
290
-	rseq_workaround_gcc_asm_size_guess();
291340 	__asm__ __volatile__ goto (
292
-		RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
341
+		RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
342
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
343
+#ifdef RSEQ_COMPARE_TWICE
344
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
345
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
346
+#endif
293347 		/* Start rseq by storing table entry pointer into rseq_cs. */
294348 		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
295349 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
....@@ -314,8 +368,8 @@
314368 		"5:\n\t"
315369 		: /* gcc asm goto does not allow outputs */
316370 		: [cpu_id]		"r" (cpu),
317
-		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
318
-		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
371
+		  [current_cpu_id]	"m" (rseq_get_abi()->cpu_id),
372
+		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
319373 		  /* try store input */
320374 		  [v2]			"m" (*v2),
321375 		  [newv2]		"r" (newv2),
....@@ -331,14 +385,11 @@
331385 		  , error1, error2
332386 #endif
333387 	);
334
-	rseq_workaround_gcc_asm_size_guess();
335388 	return 0;
336389 abort:
337
-	rseq_workaround_gcc_asm_size_guess();
338390 	RSEQ_INJECT_FAILED
339391 	return -1;
340392 cmpfail:
341
-	rseq_workaround_gcc_asm_size_guess();
342393 	return 1;
343394 #ifdef RSEQ_COMPARE_TWICE
344395 error1:
....@@ -355,9 +406,13 @@
355406 {
356407 	RSEQ_INJECT_C(9)
357408 
358
-	rseq_workaround_gcc_asm_size_guess();
359409 	__asm__ __volatile__ goto (
360
-		RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
410
+		RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
411
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
412
+#ifdef RSEQ_COMPARE_TWICE
413
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
414
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
415
+#endif
361416 		/* Start rseq by storing table entry pointer into rseq_cs. */
362417 		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
363418 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
....@@ -383,8 +438,8 @@
383438 		"5:\n\t"
384439 		: /* gcc asm goto does not allow outputs */
385440 		: [cpu_id]		"r" (cpu),
386
-		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
387
-		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
441
+		  [current_cpu_id]	"m" (rseq_get_abi()->cpu_id),
442
+		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
388443 		  /* try store input */
389444 		  [v2]			"m" (*v2),
390445 		  [newv2]		"r" (newv2),
....@@ -400,14 +455,11 @@
400455 		  , error1, error2
401456 #endif
402457 	);
403
-	rseq_workaround_gcc_asm_size_guess();
404458 	return 0;
405459 abort:
406
-	rseq_workaround_gcc_asm_size_guess();
407460 	RSEQ_INJECT_FAILED
408461 	return -1;
409462 cmpfail:
410
-	rseq_workaround_gcc_asm_size_guess();
411463 	return 1;
412464 #ifdef RSEQ_COMPARE_TWICE
413465 error1:
....@@ -424,9 +476,14 @@
424476 {
425477 	RSEQ_INJECT_C(9)
426478 
427
-	rseq_workaround_gcc_asm_size_guess();
428479 	__asm__ __volatile__ goto (
429
-		RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
480
+		RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
481
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
482
+#ifdef RSEQ_COMPARE_TWICE
483
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
484
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
485
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
486
+#endif
430487 		/* Start rseq by storing table entry pointer into rseq_cs. */
431488 		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
432489 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
....@@ -453,8 +510,8 @@
453510 		"5:\n\t"
454511 		: /* gcc asm goto does not allow outputs */
455512 		: [cpu_id]		"r" (cpu),
456
-		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
457
-		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
513
+		  [current_cpu_id]	"m" (rseq_get_abi()->cpu_id),
514
+		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
458515 		  /* cmp2 input */
459516 		  [v2]			"m" (*v2),
460517 		  [expect2]		"r" (expect2),
....@@ -470,14 +527,11 @@
470527 		  , error1, error2, error3
471528 #endif
472529 	);
473
-	rseq_workaround_gcc_asm_size_guess();
474530 	return 0;
475531 abort:
476
-	rseq_workaround_gcc_asm_size_guess();
477532 	RSEQ_INJECT_FAILED
478533 	return -1;
479534 cmpfail:
480
-	rseq_workaround_gcc_asm_size_guess();
481535 	return 1;
482536 #ifdef RSEQ_COMPARE_TWICE
483537 error1:
....@@ -498,9 +552,13 @@
498552 
499553 	RSEQ_INJECT_C(9)
500554 
501
-	rseq_workaround_gcc_asm_size_guess();
502555 	__asm__ __volatile__ goto (
503
-		RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
556
+		RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
557
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
558
+#ifdef RSEQ_COMPARE_TWICE
559
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
560
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
561
+#endif
504562 		LONG_S " %[src], %[rseq_scratch0]\n\t"
505563 		LONG_S "  %[dst], %[rseq_scratch1]\n\t"
506564 		LONG_S " %[len], %[rseq_scratch2]\n\t"
....@@ -565,8 +623,8 @@
565623 		"8:\n\t"
566624 		: /* gcc asm goto does not allow outputs */
567625 		: [cpu_id]		"r" (cpu),
568
-		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
569
-		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
626
+		  [current_cpu_id]	"m" (rseq_get_abi()->cpu_id),
627
+		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
570628 		  /* final store input */
571629 		  [v]			"m" (*v),
572630 		  [expect]		"r" (expect),
....@@ -586,21 +644,16 @@
586644 		  , error1, error2
587645 #endif
588646 	);
589
-	rseq_workaround_gcc_asm_size_guess();
590647 	return 0;
591648 abort:
592
-	rseq_workaround_gcc_asm_size_guess();
593649 	RSEQ_INJECT_FAILED
594650 	return -1;
595651 cmpfail:
596
-	rseq_workaround_gcc_asm_size_guess();
597652 	return 1;
598653 #ifdef RSEQ_COMPARE_TWICE
599654 error1:
600
-	rseq_workaround_gcc_asm_size_guess();
601655 	rseq_bug("cpu_id comparison failed");
602656 error2:
603
-	rseq_workaround_gcc_asm_size_guess();
604657 	rseq_bug("expected value comparison failed");
605658 #endif
606659 }
....@@ -614,9 +667,13 @@
614667 
615668 	RSEQ_INJECT_C(9)
616669 
617
-	rseq_workaround_gcc_asm_size_guess();
618670 	__asm__ __volatile__ goto (
619
-		RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
671
+		RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
672
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
673
+#ifdef RSEQ_COMPARE_TWICE
674
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
675
+		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
676
+#endif
620677 		LONG_S " %[src], %[rseq_scratch0]\n\t"
621678 		LONG_S " %[dst], %[rseq_scratch1]\n\t"
622679 		LONG_S " %[len], %[rseq_scratch2]\n\t"
....@@ -682,8 +739,8 @@
682739 		"8:\n\t"
683740 		: /* gcc asm goto does not allow outputs */
684741 		: [cpu_id]		"r" (cpu),
685
-		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
686
-		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
742
+		  [current_cpu_id]	"m" (rseq_get_abi()->cpu_id),
743
+		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
687744 		  /* final store input */
688745 		  [v]			"m" (*v),
689746 		  [expect]		"r" (expect),
....@@ -703,21 +760,16 @@
703760 		  , error1, error2
704761 #endif
705762 	);
706
-	rseq_workaround_gcc_asm_size_guess();
707763 	return 0;
708764 abort:
709
-	rseq_workaround_gcc_asm_size_guess();
710765 	RSEQ_INJECT_FAILED
711766 	return -1;
712767 cmpfail:
713
-	rseq_workaround_gcc_asm_size_guess();
714768 	return 1;
715769 #ifdef RSEQ_COMPARE_TWICE
716770 error1:
717
-	rseq_workaround_gcc_asm_size_guess();
718771 	rseq_bug("cpu_id comparison failed");
719772 error2:
720
-	rseq_workaround_gcc_asm_size_guess();
721773 	rseq_bug("expected value comparison failed");
722774 #endif
723775 }