kernel_5.10 no rt

hc

2023-12-11 6778948f9de86c3cfaf36725a7c87dcff9ba247f

 kernel/tools/testing/selftests/rseq/rseq-arm64.h
..
..
@@ -6,7 +6,20 @@
6
6
  * (C) Copyright 2018 - Will Deacon <will.deacon@arm.com>
7
7
  */
8
8
 
9
-#define RSEQ_SIG	0xd428bc00	/* BRK #0x45E0 */
9
+/*
10
+ * aarch64 -mbig-endian generates mixed endianness code vs data:
11
+ * little-endian code and big-endian data. Ensure the RSEQ_SIG signature
12
+ * matches code endianness.
13
+ */
14
+#define RSEQ_SIG_CODE	0xd428bc00	/* BRK #0x45E0.  */
15
+
16
+#ifdef __AARCH64EB__
17
+#define RSEQ_SIG_DATA	0x00bc28d4	/* BRK #0x45E0.  */
18
+#else
19
+#define RSEQ_SIG_DATA	RSEQ_SIG_CODE
20
+#endif
21
+
22
+#define RSEQ_SIG	RSEQ_SIG_DATA
10
23
 
11
24
 #define rseq_smp_mb()	__asm__ __volatile__ ("dmb ish" ::: "memory")
12
25
 #define rseq_smp_rmb()	__asm__ __volatile__ ("dmb ishld" ::: "memory")
..
..
@@ -82,18 +95,34 @@
82
95
 
83
96
 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip,		\
84
97
 				post_commit_offset, abort_ip)			\
85
-	"	.pushsection	__rseq_table, \"aw\"\n"				\
98
+	"	.pushsection	__rseq_cs, \"aw\"\n"				\
86
99
 	"	.balign	32\n"							\
87
100
 	__rseq_str(label) ":\n"							\
88
101
 	"	.long	" __rseq_str(version) ", " __rseq_str(flags) "\n"	\
89
102
 	"	.quad	" __rseq_str(start_ip) ", "				\
90
103
 			  __rseq_str(post_commit_offset) ", "			\
91
104
 			  __rseq_str(abort_ip) "\n"				\
105
+	"	.popsection\n\t"						\
106
+	"	.pushsection __rseq_cs_ptr_array, \"aw\"\n"				\
107
+	"	.quad " __rseq_str(label) "b\n"					\
92
108
 	"	.popsection\n"
93
109
 
94
110
 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip)	\
95
111
 	__RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,			\
96
112
 				(post_commit_ip - start_ip), abort_ip)
113
+
114
+/*
115
+ * Exit points of a rseq critical section consist of all instructions outside
116
+ * of the critical section where a critical section can either branch to or
117
+ * reach through the normal course of its execution. The abort IP and the
118
+ * post-commit IP are already part of the __rseq_cs section and should not be
119
+ * explicitly defined as additional exit points. Knowing all exit points is
120
+ * useful to assist debuggers stepping over the critical section.
121
+ */
122
+#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)				\
123
+	"	.pushsection __rseq_exit_point_array, \"aw\"\n"			\
124
+	"	.quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n"	\
125
+	"	.popsection\n"
97
126
 
98
127
 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)			\
99
128
 	RSEQ_INJECT_ASM(1)							\
..
..
@@ -105,7 +134,7 @@
105
134
 
106
135
 #define RSEQ_ASM_DEFINE_ABORT(label, abort_label)				\
107
136
 	"	b	222f\n"							\
108
-	"	.inst 	"	__rseq_str(RSEQ_SIG) "\n"			\
137
+	"	.inst 	"	__rseq_str(RSEQ_SIG_CODE) "\n"			\
109
138
 	__rseq_str(label) ":\n"							\
110
139
 	"	b	%l[" __rseq_str(abort_label) "]\n"			\
111
140
 	"222:\n"
..
..
@@ -182,6 +211,11 @@
182
211
 
183
212
 	__asm__ __volatile__ goto (
184
213
 		RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
214
+		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
215
+#ifdef RSEQ_COMPARE_TWICE
216
+		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
217
+		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
218
+#endif
185
219
 		RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
186
220
 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
187
221
 		RSEQ_INJECT_ASM(3)
..
..
@@ -196,8 +230,8 @@
196
230
 		RSEQ_ASM_DEFINE_ABORT(4, abort)
197
231
 		: /* gcc asm goto does not allow outputs */
198
232
 		: [cpu_id]		"r" (cpu),
199
-		  [current_cpu_id]	"Qo" (__rseq_abi.cpu_id),
200
-		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
233
+		  [current_cpu_id]	"Qo" (rseq_get_abi()->cpu_id),
234
+		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
201
235
 		  [v]			"Qo" (*v),
202
236
 		  [expect]		"r" (expect),
203
237
 		  [newv]		"r" (newv)
..
..
@@ -208,29 +242,38 @@
208
242
 		  , error1, error2
209
243
 #endif
210
244
 	);
211
-
245
+	rseq_after_asm_goto();
212
246
 	return 0;
213
247
 abort:
248
+	rseq_after_asm_goto();
214
249
 	RSEQ_INJECT_FAILED
215
250
 	return -1;
216
251
 cmpfail:
252
+	rseq_after_asm_goto();
217
253
 	return 1;
218
254
 #ifdef RSEQ_COMPARE_TWICE
219
255
 error1:
256
+	rseq_after_asm_goto();
220
257
 	rseq_bug("cpu_id comparison failed");
221
258
 error2:
259
+	rseq_after_asm_goto();
222
260
 	rseq_bug("expected value comparison failed");
223
261
 #endif
224
262
 }
225
263
 
226
264
 static inline __attribute__((always_inline))
227
265
 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
228
-			       off_t voffp, intptr_t *load, int cpu)
266
+			       long voffp, intptr_t *load, int cpu)
229
267
 {
230
268
 	RSEQ_INJECT_C(9)
231
269
 
232
270
 	__asm__ __volatile__ goto (
233
271
 		RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
272
+		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
273
+#ifdef RSEQ_COMPARE_TWICE
274
+		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
275
+		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
276
+#endif
234
277
 		RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
235
278
 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
236
279
 		RSEQ_INJECT_ASM(3)
..
..
@@ -248,8 +291,8 @@
248
291
 		RSEQ_ASM_DEFINE_ABORT(4, abort)
249
292
 		: /* gcc asm goto does not allow outputs */
250
293
 		: [cpu_id]		"r" (cpu),
251
-		  [current_cpu_id]	"Qo" (__rseq_abi.cpu_id),
252
-		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
294
+		  [current_cpu_id]	"Qo" (rseq_get_abi()->cpu_id),
295
+		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
253
296
 		  [v]			"Qo" (*v),
254
297
 		  [expectnot]		"r" (expectnot),
255
298
 		  [load]		"Qo" (*load),
..
..
@@ -261,16 +304,21 @@
261
304
 		  , error1, error2
262
305
 #endif
263
306
 	);
307
+	rseq_after_asm_goto();
264
308
 	return 0;
265
309
 abort:
310
+	rseq_after_asm_goto();
266
311
 	RSEQ_INJECT_FAILED
267
312
 	return -1;
268
313
 cmpfail:
314
+	rseq_after_asm_goto();
269
315
 	return 1;
270
316
 #ifdef RSEQ_COMPARE_TWICE
271
317
 error1:
318
+	rseq_after_asm_goto();
272
319
 	rseq_bug("cpu_id comparison failed");
273
320
 error2:
321
+	rseq_after_asm_goto();
274
322
 	rseq_bug("expected value comparison failed");
275
323
 #endif
276
324
 }
..
..
@@ -282,6 +330,9 @@
282
330
 
283
331
 	__asm__ __volatile__ goto (
284
332
 		RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
333
+#ifdef RSEQ_COMPARE_TWICE
334
+		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
335
+#endif
285
336
 		RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
286
337
 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
287
338
 		RSEQ_INJECT_ASM(3)
..
..
@@ -295,8 +346,8 @@
295
346
 		RSEQ_ASM_DEFINE_ABORT(4, abort)
296
347
 		: /* gcc asm goto does not allow outputs */
297
348
 		: [cpu_id]		"r" (cpu),
298
-		  [current_cpu_id]	"Qo" (__rseq_abi.cpu_id),
299
-		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
349
+		  [current_cpu_id]	"Qo" (rseq_get_abi()->cpu_id),
350
+		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
300
351
 		  [v]			"Qo" (*v),
301
352
 		  [count]		"r" (count)
302
353
 		  RSEQ_INJECT_INPUT
..
..
@@ -306,12 +357,15 @@
306
357
 		  , error1
307
358
 #endif
308
359
 	);
360
+	rseq_after_asm_goto();
309
361
 	return 0;
310
362
 abort:
363
+	rseq_after_asm_goto();
311
364
 	RSEQ_INJECT_FAILED
312
365
 	return -1;
313
366
 #ifdef RSEQ_COMPARE_TWICE
314
367
 error1:
368
+	rseq_after_asm_goto();
315
369
 	rseq_bug("cpu_id comparison failed");
316
370
 #endif
317
371
 }
..
..
@@ -325,6 +379,11 @@
325
379
 
326
380
 	__asm__ __volatile__ goto (
327
381
 		RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
382
+		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
383
+#ifdef RSEQ_COMPARE_TWICE
384
+		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
385
+		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
386
+#endif
328
387
 		RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
329
388
 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
330
389
 		RSEQ_INJECT_ASM(3)
..
..
@@ -341,8 +400,8 @@
341
400
 		RSEQ_ASM_DEFINE_ABORT(4, abort)
342
401
 		: /* gcc asm goto does not allow outputs */
343
402
 		: [cpu_id]		"r" (cpu),
344
-		  [current_cpu_id]	"Qo" (__rseq_abi.cpu_id),
345
-		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
403
+		  [current_cpu_id]	"Qo" (rseq_get_abi()->cpu_id),
404
+		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
346
405
 		  [expect]		"r" (expect),
347
406
 		  [v]			"Qo" (*v),
348
407
 		  [newv]		"r" (newv),
..
..
@@ -355,17 +414,21 @@
355
414
 		  , error1, error2
356
415
 #endif
357
416
 	);
358
-
417
+	rseq_after_asm_goto();
359
418
 	return 0;
360
419
 abort:
420
+	rseq_after_asm_goto();
361
421
 	RSEQ_INJECT_FAILED
362
422
 	return -1;
363
423
 cmpfail:
424
+	rseq_after_asm_goto();
364
425
 	return 1;
365
426
 #ifdef RSEQ_COMPARE_TWICE
366
427
 error1:
428
+	rseq_after_asm_goto();
367
429
 	rseq_bug("cpu_id comparison failed");
368
430
 error2:
431
+	rseq_after_asm_goto();
369
432
 	rseq_bug("expected value comparison failed");
370
433
 #endif
371
434
 }
..
..
@@ -379,6 +442,11 @@
379
442
 
380
443
 	__asm__ __volatile__ goto (
381
444
 		RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
445
+		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
446
+#ifdef RSEQ_COMPARE_TWICE
447
+		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
448
+		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
449
+#endif
382
450
 		RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
383
451
 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
384
452
 		RSEQ_INJECT_ASM(3)
..
..
@@ -395,8 +463,8 @@
395
463
 		RSEQ_ASM_DEFINE_ABORT(4, abort)
396
464
 		: /* gcc asm goto does not allow outputs */
397
465
 		: [cpu_id]		"r" (cpu),
398
-		  [current_cpu_id]	"Qo" (__rseq_abi.cpu_id),
399
-		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
466
+		  [current_cpu_id]	"Qo" (rseq_get_abi()->cpu_id),
467
+		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
400
468
 		  [expect]		"r" (expect),
401
469
 		  [v]			"Qo" (*v),
402
470
 		  [newv]		"r" (newv),
..
..
@@ -409,17 +477,21 @@
409
477
 		  , error1, error2
410
478
 #endif
411
479
 	);
412
-
480
+	rseq_after_asm_goto();
413
481
 	return 0;
414
482
 abort:
483
+	rseq_after_asm_goto();
415
484
 	RSEQ_INJECT_FAILED
416
485
 	return -1;
417
486
 cmpfail:
487
+	rseq_after_asm_goto();
418
488
 	return 1;
419
489
 #ifdef RSEQ_COMPARE_TWICE
420
490
 error1:
491
+	rseq_after_asm_goto();
421
492
 	rseq_bug("cpu_id comparison failed");
422
493
 error2:
494
+	rseq_after_asm_goto();
423
495
 	rseq_bug("expected value comparison failed");
424
496
 #endif
425
497
 }
..
..
@@ -433,6 +505,12 @@
433
505
 
434
506
 	__asm__ __volatile__ goto (
435
507
 		RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
508
+		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
509
+#ifdef RSEQ_COMPARE_TWICE
510
+		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
511
+		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
512
+		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error3])
513
+#endif
436
514
 		RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
437
515
 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
438
516
 		RSEQ_INJECT_ASM(3)
..
..
@@ -450,8 +528,8 @@
450
528
 		RSEQ_ASM_DEFINE_ABORT(4, abort)
451
529
 		: /* gcc asm goto does not allow outputs */
452
530
 		: [cpu_id]		"r" (cpu),
453
-		  [current_cpu_id]	"Qo" (__rseq_abi.cpu_id),
454
-		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
531
+		  [current_cpu_id]	"Qo" (rseq_get_abi()->cpu_id),
532
+		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
455
533
 		  [v]			"Qo" (*v),
456
534
 		  [expect]		"r" (expect),
457
535
 		  [v2]			"Qo" (*v2),
..
..
@@ -464,19 +542,24 @@
464
542
 		  , error1, error2, error3
465
543
 #endif
466
544
 	);
467
-
545
+	rseq_after_asm_goto();
468
546
 	return 0;
469
547
 abort:
548
+	rseq_after_asm_goto();
470
549
 	RSEQ_INJECT_FAILED
471
550
 	return -1;
472
551
 cmpfail:
552
+	rseq_after_asm_goto();
473
553
 	return 1;
474
554
 #ifdef RSEQ_COMPARE_TWICE
475
555
 error1:
556
+	rseq_after_asm_goto();
476
557
 	rseq_bug("cpu_id comparison failed");
477
558
 error2:
559
+	rseq_after_asm_goto();
478
560
 	rseq_bug("expected value comparison failed");
479
561
 error3:
562
+	rseq_after_asm_goto();
480
563
 	rseq_bug("2nd expected value comparison failed");
481
564
 #endif
482
565
 }
..
..
@@ -490,6 +573,11 @@
490
573
 
491
574
 	__asm__ __volatile__ goto (
492
575
 		RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
576
+		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
577
+#ifdef RSEQ_COMPARE_TWICE
578
+		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
579
+		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
580
+#endif
493
581
 		RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
494
582
 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
495
583
 		RSEQ_INJECT_ASM(3)
..
..
@@ -506,8 +594,8 @@
506
594
 		RSEQ_ASM_DEFINE_ABORT(4, abort)
507
595
 		: /* gcc asm goto does not allow outputs */
508
596
 		: [cpu_id]		"r" (cpu),
509
-		  [current_cpu_id]	"Qo" (__rseq_abi.cpu_id),
510
-		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
597
+		  [current_cpu_id]	"Qo" (rseq_get_abi()->cpu_id),
598
+		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
511
599
 		  [expect]		"r" (expect),
512
600
 		  [v]			"Qo" (*v),
513
601
 		  [newv]		"r" (newv),
..
..
@@ -521,17 +609,21 @@
521
609
 		  , error1, error2
522
610
 #endif
523
611
 	);
524
-
612
+	rseq_after_asm_goto();
525
613
 	return 0;
526
614
 abort:
615
+	rseq_after_asm_goto();
527
616
 	RSEQ_INJECT_FAILED
528
617
 	return -1;
529
618
 cmpfail:
619
+	rseq_after_asm_goto();
530
620
 	return 1;
531
621
 #ifdef RSEQ_COMPARE_TWICE
532
622
 error1:
623
+	rseq_after_asm_goto();
533
624
 	rseq_bug("cpu_id comparison failed");
534
625
 error2:
626
+	rseq_after_asm_goto();
535
627
 	rseq_bug("expected value comparison failed");
536
628
 #endif
537
629
 }
..
..
@@ -545,6 +637,11 @@
545
637
 
546
638
 	__asm__ __volatile__ goto (
547
639
 		RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
640
+		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
641
+#ifdef RSEQ_COMPARE_TWICE
642
+		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
643
+		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
644
+#endif
548
645
 		RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
549
646
 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
550
647
 		RSEQ_INJECT_ASM(3)
..
..
@@ -561,8 +658,8 @@
561
658
 		RSEQ_ASM_DEFINE_ABORT(4, abort)
562
659
 		: /* gcc asm goto does not allow outputs */
563
660
 		: [cpu_id]		"r" (cpu),
564
-		  [current_cpu_id]	"Qo" (__rseq_abi.cpu_id),
565
-		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
661
+		  [current_cpu_id]	"Qo" (rseq_get_abi()->cpu_id),
662
+		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
566
663
 		  [expect]		"r" (expect),
567
664
 		  [v]			"Qo" (*v),
568
665
 		  [newv]		"r" (newv),
..
..
@@ -576,17 +673,21 @@
576
673
 		  , error1, error2
577
674
 #endif
578
675
 	);
579
-
676
+	rseq_after_asm_goto();
580
677
 	return 0;
581
678
 abort:
679
+	rseq_after_asm_goto();
582
680
 	RSEQ_INJECT_FAILED
583
681
 	return -1;
584
682
 cmpfail:
683
+	rseq_after_asm_goto();
585
684
 	return 1;
586
685
 #ifdef RSEQ_COMPARE_TWICE
587
686
 error1:
687
+	rseq_after_asm_goto();
588
688
 	rseq_bug("cpu_id comparison failed");
589
689
 error2:
690
+	rseq_after_asm_goto();
590
691
 	rseq_bug("expected value comparison failed");
591
692
 #endif
592
693
 }