hc
2024-05-10 61598093bbdd283a7edc367d900f223070ead8d2
kernel/kernel/rcu/rcuscale.c
....@@ -49,8 +49,8 @@
4949 pr_alert("%s" SCALE_FLAG " %s\n", scale_type, s)
5050 #define VERBOSE_SCALEOUT_STRING(s) \
5151 do { if (verbose) pr_alert("%s" SCALE_FLAG " %s\n", scale_type, s); } while (0)
52
-#define VERBOSE_SCALEOUT_ERRSTRING(s) \
53
- do { if (verbose) pr_alert("%s" SCALE_FLAG "!!! %s\n", scale_type, s); } while (0)
52
+#define SCALEOUT_ERRSTRING(s) \
53
+ pr_alert("%s" SCALE_FLAG "!!! %s\n", scale_type, s)
5454
5555 /*
5656 * The intended use cases for the nreaders and nwriters module parameters
....@@ -372,7 +372,7 @@
372372 sched_set_fifo_low(current);
373373
374374 if (holdoff)
375
- schedule_timeout_uninterruptible(holdoff * HZ);
375
+ schedule_timeout_idle(holdoff * HZ);
376376
377377 /*
378378 * Wait until rcu_end_inkernel_boot() is called for normal GP tests
....@@ -457,7 +457,7 @@
457457 if (gp_async) {
458458 cur_ops->gp_barrier();
459459 }
460
- writer_n_durations[me] = i_max;
460
+ writer_n_durations[me] = i_max + 1;
461461 torture_kthread_stopping("rcu_scale_writer");
462462 return 0;
463463 }
....@@ -468,89 +468,6 @@
468468 pr_alert("%s" SCALE_FLAG
469469 "--- %s: nreaders=%d nwriters=%d verbose=%d shutdown=%d\n",
470470 scale_type, tag, nrealreaders, nrealwriters, verbose, shutdown);
471
-}
472
-
473
-static void
474
-rcu_scale_cleanup(void)
475
-{
476
- int i;
477
- int j;
478
- int ngps = 0;
479
- u64 *wdp;
480
- u64 *wdpp;
481
-
482
- /*
483
- * Would like warning at start, but everything is expedited
484
- * during the mid-boot phase, so have to wait till the end.
485
- */
486
- if (rcu_gp_is_expedited() && !rcu_gp_is_normal() && !gp_exp)
487
- VERBOSE_SCALEOUT_ERRSTRING("All grace periods expedited, no normal ones to measure!");
488
- if (rcu_gp_is_normal() && gp_exp)
489
- VERBOSE_SCALEOUT_ERRSTRING("All grace periods normal, no expedited ones to measure!");
490
- if (gp_exp && gp_async)
491
- VERBOSE_SCALEOUT_ERRSTRING("No expedited async GPs, so went with async!");
492
-
493
- if (torture_cleanup_begin())
494
- return;
495
- if (!cur_ops) {
496
- torture_cleanup_end();
497
- return;
498
- }
499
-
500
- if (reader_tasks) {
501
- for (i = 0; i < nrealreaders; i++)
502
- torture_stop_kthread(rcu_scale_reader,
503
- reader_tasks[i]);
504
- kfree(reader_tasks);
505
- }
506
-
507
- if (writer_tasks) {
508
- for (i = 0; i < nrealwriters; i++) {
509
- torture_stop_kthread(rcu_scale_writer,
510
- writer_tasks[i]);
511
- if (!writer_n_durations)
512
- continue;
513
- j = writer_n_durations[i];
514
- pr_alert("%s%s writer %d gps: %d\n",
515
- scale_type, SCALE_FLAG, i, j);
516
- ngps += j;
517
- }
518
- pr_alert("%s%s start: %llu end: %llu duration: %llu gps: %d batches: %ld\n",
519
- scale_type, SCALE_FLAG,
520
- t_rcu_scale_writer_started, t_rcu_scale_writer_finished,
521
- t_rcu_scale_writer_finished -
522
- t_rcu_scale_writer_started,
523
- ngps,
524
- rcuscale_seq_diff(b_rcu_gp_test_finished,
525
- b_rcu_gp_test_started));
526
- for (i = 0; i < nrealwriters; i++) {
527
- if (!writer_durations)
528
- break;
529
- if (!writer_n_durations)
530
- continue;
531
- wdpp = writer_durations[i];
532
- if (!wdpp)
533
- continue;
534
- for (j = 0; j <= writer_n_durations[i]; j++) {
535
- wdp = &wdpp[j];
536
- pr_alert("%s%s %4d writer-duration: %5d %llu\n",
537
- scale_type, SCALE_FLAG,
538
- i, j, *wdp);
539
- if (j % 100 == 0)
540
- schedule_timeout_uninterruptible(1);
541
- }
542
- kfree(writer_durations[i]);
543
- }
544
- kfree(writer_tasks);
545
- kfree(writer_durations);
546
- kfree(writer_n_durations);
547
- }
548
-
549
- /* Do torture-type-specific cleanup operations. */
550
- if (cur_ops->cleanup != NULL)
551
- cur_ops->cleanup();
552
-
553
- torture_cleanup_end();
554471 }
555472
556473 /*
....@@ -570,21 +487,6 @@
570487 nr = 1;
571488 }
572489 return nr;
573
-}
574
-
575
-/*
576
- * RCU scalability shutdown kthread. Just waits to be awakened, then shuts
577
- * down system.
578
- */
579
-static int
580
-rcu_scale_shutdown(void *arg)
581
-{
582
- wait_event(shutdown_wq,
583
- atomic_read(&n_rcu_scale_writer_finished) >= nrealwriters);
584
- smp_mb(); /* Wake before output. */
585
- rcu_scale_cleanup();
586
- kernel_power_off();
587
- return -EINVAL;
588490 }
589491
590492 /*
....@@ -693,8 +595,8 @@
693595 static int
694596 kfree_scale_shutdown(void *arg)
695597 {
696
- wait_event(shutdown_wq,
697
- atomic_read(&n_kfree_scale_thread_ended) >= kfree_nrealthreads);
598
+ wait_event_idle(shutdown_wq,
599
+ atomic_read(&n_kfree_scale_thread_ended) >= kfree_nrealthreads);
698600
699601 smp_mb(); /* Wake before output. */
700602
....@@ -746,6 +648,108 @@
746648 torture_init_end();
747649 kfree_scale_cleanup();
748650 return firsterr;
651
+}
652
+
653
+static void
654
+rcu_scale_cleanup(void)
655
+{
656
+ int i;
657
+ int j;
658
+ int ngps = 0;
659
+ u64 *wdp;
660
+ u64 *wdpp;
661
+
662
+ /*
663
+ * Would like warning at start, but everything is expedited
664
+ * during the mid-boot phase, so have to wait till the end.
665
+ */
666
+ if (rcu_gp_is_expedited() && !rcu_gp_is_normal() && !gp_exp)
667
+ SCALEOUT_ERRSTRING("All grace periods expedited, no normal ones to measure!");
668
+ if (rcu_gp_is_normal() && gp_exp)
669
+ SCALEOUT_ERRSTRING("All grace periods normal, no expedited ones to measure!");
670
+ if (gp_exp && gp_async)
671
+ SCALEOUT_ERRSTRING("No expedited async GPs, so went with async!");
672
+
673
+ if (kfree_rcu_test) {
674
+ kfree_scale_cleanup();
675
+ return;
676
+ }
677
+
678
+ if (torture_cleanup_begin())
679
+ return;
680
+ if (!cur_ops) {
681
+ torture_cleanup_end();
682
+ return;
683
+ }
684
+
685
+ if (reader_tasks) {
686
+ for (i = 0; i < nrealreaders; i++)
687
+ torture_stop_kthread(rcu_scale_reader,
688
+ reader_tasks[i]);
689
+ kfree(reader_tasks);
690
+ }
691
+
692
+ if (writer_tasks) {
693
+ for (i = 0; i < nrealwriters; i++) {
694
+ torture_stop_kthread(rcu_scale_writer,
695
+ writer_tasks[i]);
696
+ if (!writer_n_durations)
697
+ continue;
698
+ j = writer_n_durations[i];
699
+ pr_alert("%s%s writer %d gps: %d\n",
700
+ scale_type, SCALE_FLAG, i, j);
701
+ ngps += j;
702
+ }
703
+ pr_alert("%s%s start: %llu end: %llu duration: %llu gps: %d batches: %ld\n",
704
+ scale_type, SCALE_FLAG,
705
+ t_rcu_scale_writer_started, t_rcu_scale_writer_finished,
706
+ t_rcu_scale_writer_finished -
707
+ t_rcu_scale_writer_started,
708
+ ngps,
709
+ rcuscale_seq_diff(b_rcu_gp_test_finished,
710
+ b_rcu_gp_test_started));
711
+ for (i = 0; i < nrealwriters; i++) {
712
+ if (!writer_durations)
713
+ break;
714
+ if (!writer_n_durations)
715
+ continue;
716
+ wdpp = writer_durations[i];
717
+ if (!wdpp)
718
+ continue;
719
+ for (j = 0; j < writer_n_durations[i]; j++) {
720
+ wdp = &wdpp[j];
721
+ pr_alert("%s%s %4d writer-duration: %5d %llu\n",
722
+ scale_type, SCALE_FLAG,
723
+ i, j, *wdp);
724
+ if (j % 100 == 0)
725
+ schedule_timeout_uninterruptible(1);
726
+ }
727
+ kfree(writer_durations[i]);
728
+ }
729
+ kfree(writer_tasks);
730
+ kfree(writer_durations);
731
+ kfree(writer_n_durations);
732
+ }
733
+
734
+ /* Do torture-type-specific cleanup operations. */
735
+ if (cur_ops->cleanup != NULL)
736
+ cur_ops->cleanup();
737
+
738
+ torture_cleanup_end();
739
+}
740
+
741
+/*
742
+ * RCU scalability shutdown kthread. Just waits to be awakened, then shuts
743
+ * down system.
744
+ */
745
+static int
746
+rcu_scale_shutdown(void *arg)
747
+{
748
+ wait_event_idle(shutdown_wq, atomic_read(&n_rcu_scale_writer_finished) >= nrealwriters);
749
+ smp_mb(); /* Wake before output. */
750
+ rcu_scale_cleanup();
751
+ kernel_power_off();
752
+ return -EINVAL;
749753 }
750754
751755 static int __init
....@@ -803,7 +807,7 @@
803807 reader_tasks = kcalloc(nrealreaders, sizeof(reader_tasks[0]),
804808 GFP_KERNEL);
805809 if (reader_tasks == NULL) {
806
- VERBOSE_SCALEOUT_ERRSTRING("out of memory");
810
+ SCALEOUT_ERRSTRING("out of memory");
807811 firsterr = -ENOMEM;
808812 goto unwind;
809813 }
....@@ -823,7 +827,7 @@
823827 kcalloc(nrealwriters, sizeof(*writer_n_durations),
824828 GFP_KERNEL);
825829 if (!writer_tasks || !writer_durations || !writer_n_durations) {
826
- VERBOSE_SCALEOUT_ERRSTRING("out of memory");
830
+ SCALEOUT_ERRSTRING("out of memory");
827831 firsterr = -ENOMEM;
828832 goto unwind;
829833 }