hc
2023-11-07 f45e756958099c35d6afb746df1d40a1c6302cfc
kernel/kernel/rcu/srcutree.c
....@@ -38,6 +38,8 @@
3838 #include <linux/delay.h>
3939 #include <linux/module.h>
4040 #include <linux/srcu.h>
41
+#include <linux/cpu.h>
42
+#include <linux/locallock.h>
4143
4244 #include "rcu.h"
4345 #include "rcu_segcblist.h"
....@@ -461,21 +463,6 @@
461463 }
462464
463465 /*
464
- * Track online CPUs to guide callback workqueue placement.
465
- */
466
-DEFINE_PER_CPU(bool, srcu_online);
467
-
468
-void srcu_online_cpu(unsigned int cpu)
469
-{
470
- WRITE_ONCE(per_cpu(srcu_online, cpu), true);
471
-}
472
-
473
-void srcu_offline_cpu(unsigned int cpu)
474
-{
475
- WRITE_ONCE(per_cpu(srcu_online, cpu), false);
476
-}
477
-
478
-/*
479466 * Place the workqueue handler on the specified CPU if online, otherwise
480467 * just run it whereever. This is useful for placing workqueue handlers
481468 * that are to invoke the specified CPU's callbacks.
....@@ -486,12 +473,12 @@
486473 {
487474 bool ret;
488475
489
- preempt_disable();
490
- if (READ_ONCE(per_cpu(srcu_online, cpu)))
476
+ cpus_read_lock();
477
+ if (cpu_online(cpu))
491478 ret = queue_delayed_work_on(cpu, wq, dwork, delay);
492479 else
493480 ret = queue_delayed_work(wq, dwork, delay);
494
- preempt_enable();
481
+ cpus_read_unlock();
495482 return ret;
496483 }
497484
....@@ -774,6 +761,8 @@
774761 * negligible when amoritized over that time period, and the extra latency
775762 * of a needlessly non-expedited grace period is similarly negligible.
776763 */
764
+static DEFINE_LOCAL_IRQ_LOCK(sp_llock);
765
+
777766 static bool srcu_might_be_idle(struct srcu_struct *sp)
778767 {
779768 unsigned long curseq;
....@@ -782,13 +771,13 @@
782771 unsigned long t;
783772
784773 /* If the local srcu_data structure has callbacks, not idle. */
785
- local_irq_save(flags);
774
+ local_lock_irqsave(sp_llock, flags);
786775 sdp = this_cpu_ptr(sp->sda);
787776 if (rcu_segcblist_pend_cbs(&sdp->srcu_cblist)) {
788
- local_irq_restore(flags);
777
+ local_unlock_irqrestore(sp_llock, flags);
789778 return false; /* Callbacks already present, so not idle. */
790779 }
791
- local_irq_restore(flags);
780
+ local_unlock_irqrestore(sp_llock, flags);
792781
793782 /*
794783 * No local callbacks, so probabalistically probe global state.
....@@ -866,7 +855,7 @@
866855 return;
867856 }
868857 rhp->func = func;
869
- local_irq_save(flags);
858
+ local_lock_irqsave(sp_llock, flags);
870859 sdp = this_cpu_ptr(sp->sda);
871860 spin_lock_rcu_node(sdp);
872861 rcu_segcblist_enqueue(&sdp->srcu_cblist, rhp, false);
....@@ -882,7 +871,8 @@
882871 sdp->srcu_gp_seq_needed_exp = s;
883872 needexp = true;
884873 }
885
- spin_unlock_irqrestore_rcu_node(sdp, flags);
874
+ spin_unlock_rcu_node(sdp);
875
+ local_unlock_irqrestore(sp_llock, flags);
886876 if (needgp)
887877 srcu_funnel_gp_start(sp, sdp, s, do_norm);
888878 else if (needexp)