forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2024-12-19 9370bb92b2d16684ee45cf24e879c93c509162da 
 kernel/kernel/rcu/tree.c
....@@ -100,10 +100,8 @@
100100 static bool dump_tree;
101101 module_param(dump_tree, bool, 0444);
102102 /* By default, use RCU_SOFTIRQ instead of rcuc kthreads. */
103
-static bool use_softirq = !IS_ENABLED(CONFIG_PREEMPT_RT);
104
-#ifndef CONFIG_PREEMPT_RT
103
+static bool use_softirq = true;
105104 module_param(use_softirq, bool, 0444);
106
-#endif
107105 /* Control rcu_node-tree auto-balancing at boot time. */
108106 static bool rcu_fanout_exact;
109107 module_param(rcu_fanout_exact, bool, 0444);
....@@ -966,6 +964,7 @@
966964 	}
967965 	raw_spin_unlock_rcu_node(rdp->mynode);
968966 }
967
+NOKPROBE_SYMBOL(__rcu_irq_enter_check_tick);
969968 #endif /* CONFIG_NO_HZ_FULL */
970969 
971970 /**
....@@ -1159,7 +1158,7 @@
11591158 	preempt_disable_notrace();
11601159 	rdp = this_cpu_ptr(&rcu_data);
11611160 	rnp = rdp->mynode;
1162
-	if (rdp->grpmask & rcu_rnp_online_cpus(rnp))
1161
+	if (rdp->grpmask & rcu_rnp_online_cpus(rnp) || READ_ONCE(rnp->ofl_seq) & 0x1)
11631162 		ret = true;
11641163 	preempt_enable_notrace();
11651164 	return ret;
....@@ -1726,6 +1725,7 @@
17261725  */
17271726 static bool rcu_gp_init(void)
17281727 {
1728
+	unsigned long firstseq;
17291729 	unsigned long flags;
17301730 	unsigned long oldmask;
17311731 	unsigned long mask;
....@@ -1769,6 +1769,12 @@
17691769 	 */
17701770 	rcu_state.gp_state = RCU_GP_ONOFF;
17711771 	rcu_for_each_leaf_node(rnp) {
1772
+		smp_mb(); // Pair with barriers used when updating ->ofl_seq to odd values.
1773
+		firstseq = READ_ONCE(rnp->ofl_seq);
1774
+		if (firstseq & 0x1)
1775
+			while (firstseq == READ_ONCE(rnp->ofl_seq))
1776
+				schedule_timeout_idle(1);  // Can't wake unless RCU is watching.
1777
+		smp_mb(); // Pair with barriers used when updating ->ofl_seq to even values.
17721778 		raw_spin_lock(&rcu_state.ofl_lock);
17731779 		raw_spin_lock_irq_rcu_node(rnp);
17741780 		if (rnp->qsmaskinit == rnp->qsmaskinitnext &&
....@@ -2652,7 +2658,7 @@
26522658 	struct rcu_node *rnp_old = NULL;
26532659 
26542660 	/* Funnel through hierarchy to reduce memory contention. */
2655
-	rnp = __this_cpu_read(rcu_data.mynode);
2661
+	rnp = raw_cpu_read(rcu_data.mynode);
26562662 	for (; rnp != NULL; rnp = rnp->parent) {
26572663 		ret = (READ_ONCE(rcu_state.gp_flags) & RCU_GP_FLAG_FQS) ||
26582664 		       !raw_spin_trylock(&rnp->fqslock);
....@@ -3275,6 +3281,30 @@
32753281 	}
32763282 }
32773283 
3284
+static bool
3285
+need_offload_krc(struct kfree_rcu_cpu *krcp)
3286
+{
3287
+	int i;
3288
+
3289
+	for (i = 0; i < FREE_N_CHANNELS; i++)
3290
+		if (krcp->bkvhead[i])
3291
+			return true;
3292
+
3293
+	return !!krcp->head;
3294
+}
3295
+
3296
+static bool
3297
+need_wait_for_krwp_work(struct kfree_rcu_cpu_work *krwp)
3298
+{
3299
+	int i;
3300
+
3301
+	for (i = 0; i < FREE_N_CHANNELS; i++)
3302
+		if (krwp->bkvhead_free[i])
3303
+			return true;
3304
+
3305
+	return !!krwp->head_free;
3306
+}
3307
+
32783308 /*
32793309  * Schedule the kfree batch RCU work to run in workqueue context after a GP.
32803310  *
....@@ -3292,16 +3322,13 @@
32923322 	for (i = 0; i < KFREE_N_BATCHES; i++) {
32933323 		krwp = &(krcp->krw_arr[i]);
32943324 
3295
-		/*
3296
-		 * Try to detach bkvhead or head and attach it over any
3297
-		 * available corresponding free channel. It can be that
3298
-		 * a previous RCU batch is in progress, it means that
3299
-		 * immediately to queue another one is not possible so
3300
-		 * return false to tell caller to retry.
3301
-		 */
3302
-		if ((krcp->bkvhead[0] && !krwp->bkvhead_free[0]) ||
3303
-			(krcp->bkvhead[1] && !krwp->bkvhead_free[1]) ||
3304
-				(krcp->head && !krwp->head_free)) {
3325
+		// Try to detach bulk_head or head and attach it, only when
3326
+		// all channels are free.  Any channel is not free means at krwp
3327
+		// there is on-going rcu work to handle krwp's free business.
3328
+		if (need_wait_for_krwp_work(krwp))
3329
+			continue;
3330
+
3331
+		if (need_offload_krc(krcp)) {
33053332 			// Channel 1 corresponds to SLAB ptrs.
33063333 			// Channel 2 corresponds to vmalloc ptrs.
33073334 			for (j = 0; j < FREE_N_CHANNELS; j++) {
....@@ -3328,11 +3355,11 @@
33283355 			 */
33293356 			queue_rcu_work(system_wq, &krwp->rcu_work);
33303357 		}
3331
-
3332
-		// Repeat if any "free" corresponding channel is still busy.
3333
-		if (krcp->bkvhead[0] || krcp->bkvhead[1] || krcp->head)
3334
-			repeat = true;
33353358 	}
3359
+
3360
+	// Repeat if any "free" corresponding channel is still busy.
3361
+	if (need_offload_krc(krcp))
3362
+		repeat = true;
33363363 
33373364 	return !repeat;
33383365 }
....@@ -4109,6 +4136,9 @@
41094136 
41104137 	rnp = rdp->mynode;
41114138 	mask = rdp->grpmask;
4139
+	WRITE_ONCE(rnp->ofl_seq, rnp->ofl_seq + 1);
4140
+	WARN_ON_ONCE(!(rnp->ofl_seq & 0x1));
4141
+	smp_mb(); // Pair with rcu_gp_cleanup()'s ->ofl_seq barrier().
41124142 	raw_spin_lock_irqsave_rcu_node(rnp, flags);
41134143 	WRITE_ONCE(rnp->qsmaskinitnext, rnp->qsmaskinitnext | mask);
41144144 	newcpu = !(rnp->expmaskinitnext & mask);
....@@ -4126,6 +4156,9 @@
41264156 	} else {
41274157 		raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
41284158 	}
4159
+	smp_mb(); // Pair with rcu_gp_cleanup()'s ->ofl_seq barrier().
4160
+	WRITE_ONCE(rnp->ofl_seq, rnp->ofl_seq + 1);
4161
+	WARN_ON_ONCE(rnp->ofl_seq & 0x1);
41294162 	smp_mb(); /* Ensure RCU read-side usage follows above initialization. */
41304163 }
41314164 
....@@ -4152,6 +4185,9 @@
41524185 
41534186 	/* Remove outgoing CPU from mask in the leaf rcu_node structure. */
41544187 	mask = rdp->grpmask;
4188
+	WRITE_ONCE(rnp->ofl_seq, rnp->ofl_seq + 1);
4189
+	WARN_ON_ONCE(!(rnp->ofl_seq & 0x1));
4190
+	smp_mb(); // Pair with rcu_gp_cleanup()'s ->ofl_seq barrier().
41554191 	raw_spin_lock(&rcu_state.ofl_lock);
41564192 	raw_spin_lock_irqsave_rcu_node(rnp, flags); /* Enforce GP memory-order guarantee. */
41574193 	rdp->rcu_ofl_gp_seq = READ_ONCE(rcu_state.gp_seq);
....@@ -4164,6 +4200,9 @@
41644200 	WRITE_ONCE(rnp->qsmaskinitnext, rnp->qsmaskinitnext & ~mask);
41654201 	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
41664202 	raw_spin_unlock(&rcu_state.ofl_lock);
4203
+	smp_mb(); // Pair with rcu_gp_cleanup()'s ->ofl_seq barrier().
4204
+	WRITE_ONCE(rnp->ofl_seq, rnp->ofl_seq + 1);
4205
+	WARN_ON_ONCE(rnp->ofl_seq & 0x1);
41674206 
41684207 	rdp->cpu_started = false;
41694208 }