forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2023-11-06 15ade055295d13f95d49e3d99b09f3bbfb4a43e7 
 kernel/kernel/time/hrtimer.c
....@@ -150,6 +150,11 @@
150150 
151151 #define migration_base	migration_cpu_base.clock_base[0]
152152 
153
+static inline bool is_migration_base(struct hrtimer_clock_base *base)
154
+{
155
+	return base == &migration_base;
156
+}
157
+
153158 /*
154159  * We are using hashed locking: holding per_cpu(hrtimer_bases)[n].lock
155160  * means that all timers which are tied to this base via timer->base are
....@@ -273,6 +278,11 @@
273278 }
274279 
275280 #else /* CONFIG_SMP */
281
+
282
+static inline bool is_migration_base(struct hrtimer_clock_base *base)
283
+{
284
+	return false;
285
+}
276286 
277287 static inline struct hrtimer_clock_base *
278288 lock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags)
....@@ -957,6 +967,16 @@
957967 }
958968 EXPORT_SYMBOL_GPL(hrtimer_forward);
959969 
970
+void hrtimer_grab_expiry_lock(const struct hrtimer *timer)
971
+{
972
+	struct hrtimer_clock_base *base = READ_ONCE(timer->base);
973
+
974
+	if (timer->is_soft && !is_migration_base(base)) {
975
+		spin_lock(&base->cpu_base->softirq_expiry_lock);
976
+		spin_unlock(&base->cpu_base->softirq_expiry_lock);
977
+	}
978
+}
979
+
960980 /*
961981  * enqueue_hrtimer - internal function to (re)start a timer
962982  *
....@@ -1175,7 +1195,9 @@
11751195 	 * Check whether the HRTIMER_MODE_SOFT bit and hrtimer.is_soft
11761196 	 * match.
11771197 	 */
1198
+#ifndef CONFIG_PREEMPT_RT_BASE
11781199 	WARN_ON_ONCE(!(mode & HRTIMER_MODE_SOFT) ^ !timer->is_soft);
1200
+#endif
11791201 
11801202 	base = lock_hrtimer_base(timer, &flags);
11811203 
....@@ -1238,7 +1260,7 @@
12381260 
12391261 		if (ret >= 0)
12401262 			return ret;
1241
-		cpu_relax();
1263
+		hrtimer_grab_expiry_lock(timer);
12421264 	}
12431265 }
12441266 EXPORT_SYMBOL_GPL(hrtimer_cancel);
....@@ -1335,9 +1357,16 @@
13351357 static void __hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
13361358 			   enum hrtimer_mode mode)
13371359 {
1338
-	bool softtimer = !!(mode & HRTIMER_MODE_SOFT);
1339
-	int base = softtimer ? HRTIMER_MAX_CLOCK_BASES / 2 : 0;
1360
+	bool softtimer;
1361
+	int base;
13401362 	struct hrtimer_cpu_base *cpu_base;
1363
+
1364
+	softtimer = !!(mode & HRTIMER_MODE_SOFT);
1365
+#ifdef CONFIG_PREEMPT_RT_FULL
1366
+	if (!softtimer && !(mode & HRTIMER_MODE_HARD))
1367
+		softtimer = true;
1368
+#endif
1369
+	base = softtimer ? HRTIMER_MAX_CLOCK_BASES / 2 : 0;
13411370 
13421371 	memset(timer, 0, sizeof(struct hrtimer));
13431372 
....@@ -1535,6 +1564,7 @@
15351564 	unsigned long flags;
15361565 	ktime_t now;
15371566 
1567
+	spin_lock(&cpu_base->softirq_expiry_lock);
15381568 	raw_spin_lock_irqsave(&cpu_base->lock, flags);
15391569 
15401570 	now = hrtimer_update_base(cpu_base);
....@@ -1544,6 +1574,7 @@
15441574 	hrtimer_update_softirq_timer(cpu_base, true);
15451575 
15461576 	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
1577
+	spin_unlock(&cpu_base->softirq_expiry_lock);
15471578 }
15481579 
15491580 #ifdef CONFIG_HIGH_RES_TIMERS
....@@ -1715,12 +1746,51 @@
17151746 	return HRTIMER_NORESTART;
17161747 }
17171748 
1718
-void hrtimer_init_sleeper(struct hrtimer_sleeper *sl, struct task_struct *task)
1749
+static void __hrtimer_init_sleeper(struct hrtimer_sleeper *sl,
1750
+				   clockid_t clock_id,
1751
+				   enum hrtimer_mode mode,
1752
+				   struct task_struct *task)
17191753 {
1754
+#ifdef CONFIG_PREEMPT_RT_FULL
1755
+	if (!(mode & (HRTIMER_MODE_SOFT | HRTIMER_MODE_HARD))) {
1756
+		if (task_is_realtime(current) || system_state != SYSTEM_RUNNING)
1757
+			mode |= HRTIMER_MODE_HARD;
1758
+		else
1759
+			mode |= HRTIMER_MODE_SOFT;
1760
+	}
1761
+#endif
1762
+	__hrtimer_init(&sl->timer, clock_id, mode);
17201763 	sl->timer.function = hrtimer_wakeup;
17211764 	sl->task = task;
17221765 }
1766
+
1767
+/**
1768
+ * hrtimer_init_sleeper - initialize sleeper to the given clock
1769
+ * @sl:		sleeper to be initialized
1770
+ * @clock_id:	the clock to be used
1771
+ * @mode:	timer mode abs/rel
1772
+ * @task:	the task to wake up
1773
+ */
1774
+void hrtimer_init_sleeper(struct hrtimer_sleeper *sl, clockid_t clock_id,
1775
+			  enum hrtimer_mode mode, struct task_struct *task)
1776
+{
1777
+	debug_init(&sl->timer, clock_id, mode);
1778
+	__hrtimer_init_sleeper(sl, clock_id, mode, task);
1779
+
1780
+}
17231781 EXPORT_SYMBOL_GPL(hrtimer_init_sleeper);
1782
+
1783
+#ifdef CONFIG_DEBUG_OBJECTS_TIMERS
1784
+void hrtimer_init_sleeper_on_stack(struct hrtimer_sleeper *sl,
1785
+				   clockid_t clock_id,
1786
+				   enum hrtimer_mode mode,
1787
+				   struct task_struct *task)
1788
+{
1789
+	debug_object_init_on_stack(&sl->timer, &hrtimer_debug_descr);
1790
+	__hrtimer_init_sleeper(sl, clock_id, mode, task);
1791
+}
1792
+EXPORT_SYMBOL_GPL(hrtimer_init_sleeper_on_stack);
1793
+#endif
17241794 
17251795 int nanosleep_copyout(struct restart_block *restart, struct timespec64 *ts)
17261796 {
....@@ -1745,8 +1815,6 @@
17451815 {
17461816 	struct restart_block *restart;
17471817 
1748
-	hrtimer_init_sleeper(t, current);
1749
-
17501818 	do {
17511819 		set_current_state(TASK_INTERRUPTIBLE);
17521820 		hrtimer_start_expires(&t->timer, mode);
....@@ -1754,12 +1822,12 @@
17541822 		if (likely(t->task))
17551823 			freezable_schedule();
17561824 
1825
+		__set_current_state(TASK_RUNNING);
17571826 		hrtimer_cancel(&t->timer);
17581827 		mode = HRTIMER_MODE_ABS;
17591828 
17601829 	} while (t->task && !signal_pending(current));
17611830 
1762
-	__set_current_state(TASK_RUNNING);
17631831 
17641832 	if (!t->task)
17651833 		return 0;
....@@ -1783,10 +1851,9 @@
17831851 	struct hrtimer_sleeper t;
17841852 	int ret;
17851853 
1786
-	hrtimer_init_on_stack(&t.timer, restart->nanosleep.clockid,
1787
-				HRTIMER_MODE_ABS);
1854
+	hrtimer_init_sleeper_on_stack(&t, restart->nanosleep.clockid,
1855
+				      HRTIMER_MODE_ABS, current);
17881856 	hrtimer_set_expires_tv64(&t.timer, restart->nanosleep.expires);
1789
-
17901857 	ret = do_nanosleep(&t, HRTIMER_MODE_ABS);
17911858 	destroy_hrtimer_on_stack(&t.timer);
17921859 	return ret;
....@@ -1804,7 +1871,7 @@
18041871 	if (dl_task(current) || rt_task(current))
18051872 		slack = 0;
18061873 
1807
-	hrtimer_init_on_stack(&t.timer, clockid, mode);
1874
+	hrtimer_init_sleeper_on_stack(&t, clockid, mode, current);
18081875 	hrtimer_set_expires_range_ns(&t.timer, timespec64_to_ktime(*rqtp), slack);
18091876 	ret = do_nanosleep(&t, mode);
18101877 	if (ret != -ERESTART_RESTARTBLOCK)
....@@ -1864,6 +1931,38 @@
18641931 }
18651932 #endif
18661933 
1934
+#ifdef CONFIG_PREEMPT_RT_FULL
1935
+/*
1936
+ * Sleep for 1 ms in hope whoever holds what we want will let it go.
1937
+ */
1938
+void cpu_chill(void)
1939
+{
1940
+	unsigned int freeze_flag = current->flags & PF_NOFREEZE;
1941
+	struct task_struct *self = current;
1942
+	ktime_t chill_time;
1943
+
1944
+	raw_spin_lock_irq(&self->pi_lock);
1945
+	self->saved_state = self->state;
1946
+	__set_current_state_no_track(TASK_UNINTERRUPTIBLE);
1947
+	raw_spin_unlock_irq(&self->pi_lock);
1948
+
1949
+	chill_time = ktime_set(0, NSEC_PER_MSEC);
1950
+
1951
+	current->flags |= PF_NOFREEZE;
1952
+	sleeping_lock_inc();
1953
+	schedule_hrtimeout(&chill_time, HRTIMER_MODE_REL_HARD);
1954
+	sleeping_lock_dec();
1955
+	if (!freeze_flag)
1956
+		current->flags &= ~PF_NOFREEZE;
1957
+
1958
+	raw_spin_lock_irq(&self->pi_lock);
1959
+	__set_current_state_no_track(self->saved_state);
1960
+	self->saved_state = TASK_RUNNING;
1961
+	raw_spin_unlock_irq(&self->pi_lock);
1962
+}
1963
+EXPORT_SYMBOL(cpu_chill);
1964
+#endif
1965
+
18671966 /*
18681967  * Functions related to boot-time initialization:
18691968  */
....@@ -1885,6 +1984,7 @@
18851984 	cpu_base->softirq_next_timer = NULL;
18861985 	cpu_base->expires_next = KTIME_MAX;
18871986 	cpu_base->softirq_expires_next = KTIME_MAX;
1987
+	spin_lock_init(&cpu_base->softirq_expiry_lock);
18881988 	return 0;
18891989 }
18901990 
....@@ -2003,10 +2103,8 @@
20032103 		return -EINTR;
20042104 	}
20052105 
2006
-	hrtimer_init_on_stack(&t.timer, clock_id, mode);
2106
+	hrtimer_init_sleeper_on_stack(&t, clock_id, mode, current);
20072107 	hrtimer_set_expires_range_ns(&t.timer, *expires, delta);
2008
-
2009
-	hrtimer_init_sleeper(&t, current);
20102108 
20112109 	hrtimer_start_expires(&t.timer, mode);
20122110