.. | .. |
---|
49 | 49 | pr_alert("%s" SCALE_FLAG " %s\n", scale_type, s) |
---|
50 | 50 | #define VERBOSE_SCALEOUT_STRING(s) \ |
---|
51 | 51 | do { if (verbose) pr_alert("%s" SCALE_FLAG " %s\n", scale_type, s); } while (0) |
---|
52 | | -#define VERBOSE_SCALEOUT_ERRSTRING(s) \ |
---|
53 | | - do { if (verbose) pr_alert("%s" SCALE_FLAG "!!! %s\n", scale_type, s); } while (0) |
---|
| 52 | +#define SCALEOUT_ERRSTRING(s) \ |
---|
| 53 | + pr_alert("%s" SCALE_FLAG "!!! %s\n", scale_type, s) |
---|
54 | 54 | |
---|
55 | 55 | /* |
---|
56 | 56 | * The intended use cases for the nreaders and nwriters module parameters |
---|
.. | .. |
---|
372 | 372 | sched_set_fifo_low(current); |
---|
373 | 373 | |
---|
374 | 374 | if (holdoff) |
---|
375 | | - schedule_timeout_uninterruptible(holdoff * HZ); |
---|
| 375 | + schedule_timeout_idle(holdoff * HZ); |
---|
376 | 376 | |
---|
377 | 377 | /* |
---|
378 | 378 | * Wait until rcu_end_inkernel_boot() is called for normal GP tests |
---|
.. | .. |
---|
457 | 457 | if (gp_async) { |
---|
458 | 458 | cur_ops->gp_barrier(); |
---|
459 | 459 | } |
---|
460 | | - writer_n_durations[me] = i_max; |
---|
| 460 | + writer_n_durations[me] = i_max + 1; |
---|
461 | 461 | torture_kthread_stopping("rcu_scale_writer"); |
---|
462 | 462 | return 0; |
---|
463 | 463 | } |
---|
.. | .. |
---|
468 | 468 | pr_alert("%s" SCALE_FLAG |
---|
469 | 469 | "--- %s: nreaders=%d nwriters=%d verbose=%d shutdown=%d\n", |
---|
470 | 470 | scale_type, tag, nrealreaders, nrealwriters, verbose, shutdown); |
---|
471 | | -} |
---|
472 | | - |
---|
473 | | -static void |
---|
474 | | -rcu_scale_cleanup(void) |
---|
475 | | -{ |
---|
476 | | - int i; |
---|
477 | | - int j; |
---|
478 | | - int ngps = 0; |
---|
479 | | - u64 *wdp; |
---|
480 | | - u64 *wdpp; |
---|
481 | | - |
---|
482 | | - /* |
---|
483 | | - * Would like warning at start, but everything is expedited |
---|
484 | | - * during the mid-boot phase, so have to wait till the end. |
---|
485 | | - */ |
---|
486 | | - if (rcu_gp_is_expedited() && !rcu_gp_is_normal() && !gp_exp) |
---|
487 | | - VERBOSE_SCALEOUT_ERRSTRING("All grace periods expedited, no normal ones to measure!"); |
---|
488 | | - if (rcu_gp_is_normal() && gp_exp) |
---|
489 | | - VERBOSE_SCALEOUT_ERRSTRING("All grace periods normal, no expedited ones to measure!"); |
---|
490 | | - if (gp_exp && gp_async) |
---|
491 | | - VERBOSE_SCALEOUT_ERRSTRING("No expedited async GPs, so went with async!"); |
---|
492 | | - |
---|
493 | | - if (torture_cleanup_begin()) |
---|
494 | | - return; |
---|
495 | | - if (!cur_ops) { |
---|
496 | | - torture_cleanup_end(); |
---|
497 | | - return; |
---|
498 | | - } |
---|
499 | | - |
---|
500 | | - if (reader_tasks) { |
---|
501 | | - for (i = 0; i < nrealreaders; i++) |
---|
502 | | - torture_stop_kthread(rcu_scale_reader, |
---|
503 | | - reader_tasks[i]); |
---|
504 | | - kfree(reader_tasks); |
---|
505 | | - } |
---|
506 | | - |
---|
507 | | - if (writer_tasks) { |
---|
508 | | - for (i = 0; i < nrealwriters; i++) { |
---|
509 | | - torture_stop_kthread(rcu_scale_writer, |
---|
510 | | - writer_tasks[i]); |
---|
511 | | - if (!writer_n_durations) |
---|
512 | | - continue; |
---|
513 | | - j = writer_n_durations[i]; |
---|
514 | | - pr_alert("%s%s writer %d gps: %d\n", |
---|
515 | | - scale_type, SCALE_FLAG, i, j); |
---|
516 | | - ngps += j; |
---|
517 | | - } |
---|
518 | | - pr_alert("%s%s start: %llu end: %llu duration: %llu gps: %d batches: %ld\n", |
---|
519 | | - scale_type, SCALE_FLAG, |
---|
520 | | - t_rcu_scale_writer_started, t_rcu_scale_writer_finished, |
---|
521 | | - t_rcu_scale_writer_finished - |
---|
522 | | - t_rcu_scale_writer_started, |
---|
523 | | - ngps, |
---|
524 | | - rcuscale_seq_diff(b_rcu_gp_test_finished, |
---|
525 | | - b_rcu_gp_test_started)); |
---|
526 | | - for (i = 0; i < nrealwriters; i++) { |
---|
527 | | - if (!writer_durations) |
---|
528 | | - break; |
---|
529 | | - if (!writer_n_durations) |
---|
530 | | - continue; |
---|
531 | | - wdpp = writer_durations[i]; |
---|
532 | | - if (!wdpp) |
---|
533 | | - continue; |
---|
534 | | - for (j = 0; j <= writer_n_durations[i]; j++) { |
---|
535 | | - wdp = &wdpp[j]; |
---|
536 | | - pr_alert("%s%s %4d writer-duration: %5d %llu\n", |
---|
537 | | - scale_type, SCALE_FLAG, |
---|
538 | | - i, j, *wdp); |
---|
539 | | - if (j % 100 == 0) |
---|
540 | | - schedule_timeout_uninterruptible(1); |
---|
541 | | - } |
---|
542 | | - kfree(writer_durations[i]); |
---|
543 | | - } |
---|
544 | | - kfree(writer_tasks); |
---|
545 | | - kfree(writer_durations); |
---|
546 | | - kfree(writer_n_durations); |
---|
547 | | - } |
---|
548 | | - |
---|
549 | | - /* Do torture-type-specific cleanup operations. */ |
---|
550 | | - if (cur_ops->cleanup != NULL) |
---|
551 | | - cur_ops->cleanup(); |
---|
552 | | - |
---|
553 | | - torture_cleanup_end(); |
---|
554 | 471 | } |
---|
555 | 472 | |
---|
556 | 473 | /* |
---|
.. | .. |
---|
570 | 487 | nr = 1; |
---|
571 | 488 | } |
---|
572 | 489 | return nr; |
---|
573 | | -} |
---|
574 | | - |
---|
575 | | -/* |
---|
576 | | - * RCU scalability shutdown kthread. Just waits to be awakened, then shuts |
---|
577 | | - * down system. |
---|
578 | | - */ |
---|
579 | | -static int |
---|
580 | | -rcu_scale_shutdown(void *arg) |
---|
581 | | -{ |
---|
582 | | - wait_event(shutdown_wq, |
---|
583 | | - atomic_read(&n_rcu_scale_writer_finished) >= nrealwriters); |
---|
584 | | - smp_mb(); /* Wake before output. */ |
---|
585 | | - rcu_scale_cleanup(); |
---|
586 | | - kernel_power_off(); |
---|
587 | | - return -EINVAL; |
---|
588 | 490 | } |
---|
589 | 491 | |
---|
590 | 492 | /* |
---|
.. | .. |
---|
693 | 595 | static int |
---|
694 | 596 | kfree_scale_shutdown(void *arg) |
---|
695 | 597 | { |
---|
696 | | - wait_event(shutdown_wq, |
---|
697 | | - atomic_read(&n_kfree_scale_thread_ended) >= kfree_nrealthreads); |
---|
| 598 | + wait_event_idle(shutdown_wq, |
---|
| 599 | + atomic_read(&n_kfree_scale_thread_ended) >= kfree_nrealthreads); |
---|
698 | 600 | |
---|
699 | 601 | smp_mb(); /* Wake before output. */ |
---|
700 | 602 | |
---|
.. | .. |
---|
746 | 648 | torture_init_end(); |
---|
747 | 649 | kfree_scale_cleanup(); |
---|
748 | 650 | return firsterr; |
---|
| 651 | +} |
---|
| 652 | + |
---|
| 653 | +static void |
---|
| 654 | +rcu_scale_cleanup(void) |
---|
| 655 | +{ |
---|
| 656 | + int i; |
---|
| 657 | + int j; |
---|
| 658 | + int ngps = 0; |
---|
| 659 | + u64 *wdp; |
---|
| 660 | + u64 *wdpp; |
---|
| 661 | + |
---|
| 662 | + /* |
---|
| 663 | + * Would like warning at start, but everything is expedited |
---|
| 664 | + * during the mid-boot phase, so have to wait till the end. |
---|
| 665 | + */ |
---|
| 666 | + if (rcu_gp_is_expedited() && !rcu_gp_is_normal() && !gp_exp) |
---|
| 667 | + SCALEOUT_ERRSTRING("All grace periods expedited, no normal ones to measure!"); |
---|
| 668 | + if (rcu_gp_is_normal() && gp_exp) |
---|
| 669 | + SCALEOUT_ERRSTRING("All grace periods normal, no expedited ones to measure!"); |
---|
| 670 | + if (gp_exp && gp_async) |
---|
| 671 | + SCALEOUT_ERRSTRING("No expedited async GPs, so went with async!"); |
---|
| 672 | + |
---|
| 673 | + if (kfree_rcu_test) { |
---|
| 674 | + kfree_scale_cleanup(); |
---|
| 675 | + return; |
---|
| 676 | + } |
---|
| 677 | + |
---|
| 678 | + if (torture_cleanup_begin()) |
---|
| 679 | + return; |
---|
| 680 | + if (!cur_ops) { |
---|
| 681 | + torture_cleanup_end(); |
---|
| 682 | + return; |
---|
| 683 | + } |
---|
| 684 | + |
---|
| 685 | + if (reader_tasks) { |
---|
| 686 | + for (i = 0; i < nrealreaders; i++) |
---|
| 687 | + torture_stop_kthread(rcu_scale_reader, |
---|
| 688 | + reader_tasks[i]); |
---|
| 689 | + kfree(reader_tasks); |
---|
| 690 | + } |
---|
| 691 | + |
---|
| 692 | + if (writer_tasks) { |
---|
| 693 | + for (i = 0; i < nrealwriters; i++) { |
---|
| 694 | + torture_stop_kthread(rcu_scale_writer, |
---|
| 695 | + writer_tasks[i]); |
---|
| 696 | + if (!writer_n_durations) |
---|
| 697 | + continue; |
---|
| 698 | + j = writer_n_durations[i]; |
---|
| 699 | + pr_alert("%s%s writer %d gps: %d\n", |
---|
| 700 | + scale_type, SCALE_FLAG, i, j); |
---|
| 701 | + ngps += j; |
---|
| 702 | + } |
---|
| 703 | + pr_alert("%s%s start: %llu end: %llu duration: %llu gps: %d batches: %ld\n", |
---|
| 704 | + scale_type, SCALE_FLAG, |
---|
| 705 | + t_rcu_scale_writer_started, t_rcu_scale_writer_finished, |
---|
| 706 | + t_rcu_scale_writer_finished - |
---|
| 707 | + t_rcu_scale_writer_started, |
---|
| 708 | + ngps, |
---|
| 709 | + rcuscale_seq_diff(b_rcu_gp_test_finished, |
---|
| 710 | + b_rcu_gp_test_started)); |
---|
| 711 | + for (i = 0; i < nrealwriters; i++) { |
---|
| 712 | + if (!writer_durations) |
---|
| 713 | + break; |
---|
| 714 | + if (!writer_n_durations) |
---|
| 715 | + continue; |
---|
| 716 | + wdpp = writer_durations[i]; |
---|
| 717 | + if (!wdpp) |
---|
| 718 | + continue; |
---|
| 719 | + for (j = 0; j < writer_n_durations[i]; j++) { |
---|
| 720 | + wdp = &wdpp[j]; |
---|
| 721 | + pr_alert("%s%s %4d writer-duration: %5d %llu\n", |
---|
| 722 | + scale_type, SCALE_FLAG, |
---|
| 723 | + i, j, *wdp); |
---|
| 724 | + if (j % 100 == 0) |
---|
| 725 | + schedule_timeout_uninterruptible(1); |
---|
| 726 | + } |
---|
| 727 | + kfree(writer_durations[i]); |
---|
| 728 | + } |
---|
| 729 | + kfree(writer_tasks); |
---|
| 730 | + kfree(writer_durations); |
---|
| 731 | + kfree(writer_n_durations); |
---|
| 732 | + } |
---|
| 733 | + |
---|
| 734 | + /* Do torture-type-specific cleanup operations. */ |
---|
| 735 | + if (cur_ops->cleanup != NULL) |
---|
| 736 | + cur_ops->cleanup(); |
---|
| 737 | + |
---|
| 738 | + torture_cleanup_end(); |
---|
| 739 | +} |
---|
| 740 | + |
---|
| 741 | +/* |
---|
| 742 | + * RCU scalability shutdown kthread. Just waits to be awakened, then shuts |
---|
| 743 | + * down system. |
---|
| 744 | + */ |
---|
| 745 | +static int |
---|
| 746 | +rcu_scale_shutdown(void *arg) |
---|
| 747 | +{ |
---|
| 748 | + wait_event_idle(shutdown_wq, atomic_read(&n_rcu_scale_writer_finished) >= nrealwriters); |
---|
| 749 | + smp_mb(); /* Wake before output. */ |
---|
| 750 | + rcu_scale_cleanup(); |
---|
| 751 | + kernel_power_off(); |
---|
| 752 | + return -EINVAL; |
---|
749 | 753 | } |
---|
750 | 754 | |
---|
751 | 755 | static int __init |
---|
.. | .. |
---|
803 | 807 | reader_tasks = kcalloc(nrealreaders, sizeof(reader_tasks[0]), |
---|
804 | 808 | GFP_KERNEL); |
---|
805 | 809 | if (reader_tasks == NULL) { |
---|
806 | | - VERBOSE_SCALEOUT_ERRSTRING("out of memory"); |
---|
| 810 | + SCALEOUT_ERRSTRING("out of memory"); |
---|
807 | 811 | firsterr = -ENOMEM; |
---|
808 | 812 | goto unwind; |
---|
809 | 813 | } |
---|
.. | .. |
---|
823 | 827 | kcalloc(nrealwriters, sizeof(*writer_n_durations), |
---|
824 | 828 | GFP_KERNEL); |
---|
825 | 829 | if (!writer_tasks || !writer_durations || !writer_n_durations) { |
---|
826 | | - VERBOSE_SCALEOUT_ERRSTRING("out of memory"); |
---|
| 830 | + SCALEOUT_ERRSTRING("out of memory"); |
---|
827 | 831 | firsterr = -ENOMEM; |
---|
828 | 832 | goto unwind; |
---|
829 | 833 | } |
---|