forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2024-12-19 9370bb92b2d16684ee45cf24e879c93c509162da 
 kernel/kernel/time/hrtimer.c
....@@ -1,34 +1,25 @@
1
+// SPDX-License-Identifier: GPL-2.0
12 /*
2
- *  linux/kernel/hrtimer.c
3
- *
43  *  Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
54  *  Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
65  *  Copyright(C) 2006-2007  Timesys Corp., Thomas Gleixner
76  *
87  *  High-resolution kernel timers
98  *
10
- *  In contrast to the low-resolution timeout API implemented in
11
- *  kernel/timer.c, hrtimers provide finer resolution and accuracy
12
- *  depending on system configuration and capabilities.
13
- *
14
- *  These timers are currently used for:
15
- *   - itimers
16
- *   - POSIX timers
17
- *   - nanosleep
18
- *   - precise in-kernel timing
9
+ *  In contrast to the low-resolution timeout API, aka timer wheel,
10
+ *  hrtimers provide finer resolution and accuracy depending on system
11
+ *  configuration and capabilities.
1912  *
2013  *  Started by: Thomas Gleixner and Ingo Molnar
2114  *
2215  *  Credits:
23
- *	based on kernel/timer.c
16
+ *	Based on the original timer wheel code
2417  *
2518  *	Help, testing, suggestions, bugfixes, improvements were
2619  *	provided by:
2720  *
2821  *	George Anzinger, Andrew Morton, Steven Rostedt, Roman Zippel
2922  *	et. al.
30
- *
31
- *  For licencing details see kernel-base/COPYING
3223  */
3324 
3425 #include <linux/cpu.h>
....@@ -39,7 +30,6 @@
3930 #include <linux/syscalls.h>
4031 #include <linux/interrupt.h>
4132 #include <linux/tick.h>
42
-#include <linux/seq_file.h>
4333 #include <linux/err.h>
4434 #include <linux/debugobjects.h>
4535 #include <linux/sched/signal.h>
....@@ -145,7 +135,11 @@
145135  * timer->base->cpu_base
146136  */
147137 static struct hrtimer_cpu_base migration_cpu_base = {
148
-	.clock_base = { { .cpu_base = &migration_cpu_base, }, },
138
+	.clock_base = { {
139
+		.cpu_base = &migration_cpu_base,
140
+		.seq      = SEQCNT_RAW_SPINLOCK_ZERO(migration_cpu_base.seq,
141
+						     &migration_cpu_base.lock),
142
+	}, },
149143 };
150144 
151145 #define migration_base	migration_cpu_base.clock_base[0]
....@@ -174,7 +168,7 @@
174168 	struct hrtimer_clock_base *base;
175169 
176170 	for (;;) {
177
-		base = timer->base;
171
+		base = READ_ONCE(timer->base);
178172 		if (likely(base != &migration_base)) {
179173 			raw_spin_lock_irqsave(&base->cpu_base->lock, *flags);
180174 			if (likely(base == timer->base))
....@@ -254,7 +248,7 @@
254248 			return base;
255249 
256250 		/* See the comment in lock_hrtimer_base() */
257
-		timer->base = &migration_base;
251
+		WRITE_ONCE(timer->base, &migration_base);
258252 		raw_spin_unlock(&base->cpu_base->lock);
259253 		raw_spin_lock(&new_base->cpu_base->lock);
260254 
....@@ -263,10 +257,10 @@
263257 			raw_spin_unlock(&new_base->cpu_base->lock);
264258 			raw_spin_lock(&base->cpu_base->lock);
265259 			new_cpu_base = this_cpu_base;
266
-			timer->base = base;
260
+			WRITE_ONCE(timer->base, base);
267261 			goto again;
268262 		}
269
-		timer->base = new_base;
263
+		WRITE_ONCE(timer->base, new_base);
270264 	} else {
271265 		if (new_cpu_base != this_cpu_base &&
272266 		    hrtimer_check_target(timer, new_base)) {
....@@ -321,7 +315,7 @@
321315 		div >>= 1;
322316 	}
323317 	tmp >>= sft;
324
-	do_div(tmp, (unsigned long) div);
318
+	do_div(tmp, (u32) div);
325319 	return dclc < 0 ? -tmp : tmp;
326320 }
327321 EXPORT_SYMBOL_GPL(__ktime_divns);
....@@ -348,7 +342,7 @@
348342 
349343 #ifdef CONFIG_DEBUG_OBJECTS_TIMERS
350344 
351
-static struct debug_obj_descr hrtimer_debug_descr;
345
+static const struct debug_obj_descr hrtimer_debug_descr;
352346 
353347 static void *hrtimer_debug_hint(void *addr)
354348 {
....@@ -383,7 +377,7 @@
383377 	switch (state) {
384378 	case ODEBUG_STATE_ACTIVE:
385379 		WARN_ON(1);
386
-
380
+		fallthrough;
387381 	default:
388382 		return false;
389383 	}
....@@ -407,7 +401,7 @@
407401 	}
408402 }
409403 
410
-static struct debug_obj_descr hrtimer_debug_descr = {
404
+static const struct debug_obj_descr hrtimer_debug_descr = {
411405 	.name		= "hrtimer",
412406 	.debug_hint	= hrtimer_debug_hint,
413407 	.fixup_init	= hrtimer_fixup_init,
....@@ -431,11 +425,6 @@
431425 	debug_object_deactivate(timer, &hrtimer_debug_descr);
432426 }
433427 
434
-static inline void debug_hrtimer_free(struct hrtimer *timer)
435
-{
436
-	debug_object_free(timer, &hrtimer_debug_descr);
437
-}
438
-
439428 static void __hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
440429 			   enum hrtimer_mode mode);
441430 
....@@ -446,6 +435,17 @@
446435 	__hrtimer_init(timer, clock_id, mode);
447436 }
448437 EXPORT_SYMBOL_GPL(hrtimer_init_on_stack);
438
+
439
+static void __hrtimer_init_sleeper(struct hrtimer_sleeper *sl,
440
+				   clockid_t clock_id, enum hrtimer_mode mode);
441
+
442
+void hrtimer_init_sleeper_on_stack(struct hrtimer_sleeper *sl,
443
+				   clockid_t clock_id, enum hrtimer_mode mode)
444
+{
445
+	debug_object_init_on_stack(&sl->timer, &hrtimer_debug_descr);
446
+	__hrtimer_init_sleeper(sl, clock_id, mode);
447
+}
448
+EXPORT_SYMBOL_GPL(hrtimer_init_sleeper_on_stack);
449449 
450450 void destroy_hrtimer_on_stack(struct hrtimer *timer)
451451 {
....@@ -758,22 +758,6 @@
758758 	retrigger_next_event(NULL);
759759 }
760760 
761
-static void clock_was_set_work(struct work_struct *work)
762
-{
763
-	clock_was_set();
764
-}
765
-
766
-static DECLARE_WORK(hrtimer_work, clock_was_set_work);
767
-
768
-/*
769
- * Called from timekeeping and resume code to reprogram the hrtimer
770
- * interrupt device on all cpus.
771
- */
772
-void clock_was_set_delayed(void)
773
-{
774
-	schedule_work(&hrtimer_work);
775
-}
776
-
777761 #else
778762 
779763 static inline int hrtimer_is_hres_enabled(void) { return 0; }
....@@ -891,6 +875,22 @@
891875 	timerfd_clock_was_set();
892876 }
893877 
878
+static void clock_was_set_work(struct work_struct *work)
879
+{
880
+	clock_was_set();
881
+}
882
+
883
+static DECLARE_WORK(hrtimer_work, clock_was_set_work);
884
+
885
+/*
886
+ * Called from timekeeping and resume code to reprogram the hrtimer
887
+ * interrupt device on all cpus and to notify timerfd.
888
+ */
889
+void clock_was_set_delayed(void)
890
+{
891
+	schedule_work(&hrtimer_work);
892
+}
893
+
894894 /*
895895  * During resume we might have to reprogram the high resolution timer
896896  * interrupt on all online CPUs.  However, all other CPUs will be
....@@ -966,16 +966,6 @@
966966 	return orun;
967967 }
968968 EXPORT_SYMBOL_GPL(hrtimer_forward);
969
-
970
-void hrtimer_grab_expiry_lock(const struct hrtimer *timer)
971
-{
972
-	struct hrtimer_clock_base *base = READ_ONCE(timer->base);
973
-
974
-	if (timer->is_soft && !is_migration_base(base)) {
975
-		spin_lock(&base->cpu_base->softirq_expiry_lock);
976
-		spin_unlock(&base->cpu_base->softirq_expiry_lock);
977
-	}
978
-}
979969 
980970 /*
981971  * enqueue_hrtimer - internal function to (re)start a timer
....@@ -1193,11 +1183,13 @@
11931183 
11941184 	/*
11951185 	 * Check whether the HRTIMER_MODE_SOFT bit and hrtimer.is_soft
1196
-	 * match.
1186
+	 * match on CONFIG_PREEMPT_RT = n. With PREEMPT_RT check the hard
1187
+	 * expiry mode because unmarked timers are moved to softirq expiry.
11971188 	 */
1198
-#ifndef CONFIG_PREEMPT_RT_BASE
1199
-	WARN_ON_ONCE(!(mode & HRTIMER_MODE_SOFT) ^ !timer->is_soft);
1200
-#endif
1189
+	if (!IS_ENABLED(CONFIG_PREEMPT_RT))
1190
+		WARN_ON_ONCE(!(mode & HRTIMER_MODE_SOFT) ^ !timer->is_soft);
1191
+	else
1192
+		WARN_ON_ONCE(!(mode & HRTIMER_MODE_HARD) ^ !timer->is_hard);
12011193 
12021194 	base = lock_hrtimer_base(timer, &flags);
12031195 
....@@ -1213,9 +1205,10 @@
12131205  * @timer:	hrtimer to stop
12141206  *
12151207  * Returns:
1216
- *  0 when the timer was not active
1217
- *  1 when the timer was active
1218
- * -1 when the timer is currently executing the callback function and
1208
+ *
1209
+ *  *  0 when the timer was not active
1210
+ *  *  1 when the timer was active
1211
+ *  * -1 when the timer is currently executing the callback function and
12191212  *    cannot be stopped
12201213  */
12211214 int hrtimer_try_to_cancel(struct hrtimer *timer)
....@@ -1245,6 +1238,93 @@
12451238 }
12461239 EXPORT_SYMBOL_GPL(hrtimer_try_to_cancel);
12471240 
1241
+#ifdef CONFIG_PREEMPT_RT
1242
+static void hrtimer_cpu_base_init_expiry_lock(struct hrtimer_cpu_base *base)
1243
+{
1244
+	spin_lock_init(&base->softirq_expiry_lock);
1245
+}
1246
+
1247
+static void hrtimer_cpu_base_lock_expiry(struct hrtimer_cpu_base *base)
1248
+{
1249
+	spin_lock(&base->softirq_expiry_lock);
1250
+}
1251
+
1252
+static void hrtimer_cpu_base_unlock_expiry(struct hrtimer_cpu_base *base)
1253
+{
1254
+	spin_unlock(&base->softirq_expiry_lock);
1255
+}
1256
+
1257
+/*
1258
+ * The counterpart to hrtimer_cancel_wait_running().
1259
+ *
1260
+ * If there is a waiter for cpu_base->expiry_lock, then it was waiting for
1261
+ * the timer callback to finish. Drop expiry_lock and reaquire it. That
1262
+ * allows the waiter to acquire the lock and make progress.
1263
+ */
1264
+static void hrtimer_sync_wait_running(struct hrtimer_cpu_base *cpu_base,
1265
+				      unsigned long flags)
1266
+{
1267
+	if (atomic_read(&cpu_base->timer_waiters)) {
1268
+		raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
1269
+		spin_unlock(&cpu_base->softirq_expiry_lock);
1270
+		spin_lock(&cpu_base->softirq_expiry_lock);
1271
+		raw_spin_lock_irq(&cpu_base->lock);
1272
+	}
1273
+}
1274
+
1275
+/*
1276
+ * This function is called on PREEMPT_RT kernels when the fast path
1277
+ * deletion of a timer failed because the timer callback function was
1278
+ * running.
1279
+ *
1280
+ * This prevents priority inversion: if the soft irq thread is preempted
1281
+ * in the middle of a timer callback, then calling del_timer_sync() can
1282
+ * lead to two issues:
1283
+ *
1284
+ *  - If the caller is on a remote CPU then it has to spin wait for the timer
1285
+ *    handler to complete. This can result in unbound priority inversion.
1286
+ *
1287
+ *  - If the caller originates from the task which preempted the timer
1288
+ *    handler on the same CPU, then spin waiting for the timer handler to
1289
+ *    complete is never going to end.
1290
+ */
1291
+void hrtimer_cancel_wait_running(const struct hrtimer *timer)
1292
+{
1293
+	/* Lockless read. Prevent the compiler from reloading it below */
1294
+	struct hrtimer_clock_base *base = READ_ONCE(timer->base);
1295
+
1296
+	/*
1297
+	 * Just relax if the timer expires in hard interrupt context or if
1298
+	 * it is currently on the migration base.
1299
+	 */
1300
+	if (!timer->is_soft || is_migration_base(base)) {
1301
+		cpu_relax();
1302
+		return;
1303
+	}
1304
+
1305
+	/*
1306
+	 * Mark the base as contended and grab the expiry lock, which is
1307
+	 * held by the softirq across the timer callback. Drop the lock
1308
+	 * immediately so the softirq can expire the next timer. In theory
1309
+	 * the timer could already be running again, but that's more than
1310
+	 * unlikely and just causes another wait loop.
1311
+	 */
1312
+	atomic_inc(&base->cpu_base->timer_waiters);
1313
+	spin_lock_bh(&base->cpu_base->softirq_expiry_lock);
1314
+	atomic_dec(&base->cpu_base->timer_waiters);
1315
+	spin_unlock_bh(&base->cpu_base->softirq_expiry_lock);
1316
+}
1317
+#else
1318
+static inline void
1319
+hrtimer_cpu_base_init_expiry_lock(struct hrtimer_cpu_base *base) { }
1320
+static inline void
1321
+hrtimer_cpu_base_lock_expiry(struct hrtimer_cpu_base *base) { }
1322
+static inline void
1323
+hrtimer_cpu_base_unlock_expiry(struct hrtimer_cpu_base *base) { }
1324
+static inline void hrtimer_sync_wait_running(struct hrtimer_cpu_base *base,
1325
+					     unsigned long flags) { }
1326
+#endif
1327
+
12481328 /**
12491329  * hrtimer_cancel - cancel a timer and wait for the handler to finish.
12501330  * @timer:	the timer to be cancelled
....@@ -1255,13 +1335,15 @@
12551335  */
12561336 int hrtimer_cancel(struct hrtimer *timer)
12571337 {
1258
-	for (;;) {
1259
-		int ret = hrtimer_try_to_cancel(timer);
1338
+	int ret;
12601339 
1261
-		if (ret >= 0)
1262
-			return ret;
1263
-		hrtimer_grab_expiry_lock(timer);
1264
-	}
1340
+	do {
1341
+		ret = hrtimer_try_to_cancel(timer);
1342
+
1343
+		if (ret < 0)
1344
+			hrtimer_cancel_wait_running(timer);
1345
+	} while (ret < 0);
1346
+	return ret;
12651347 }
12661348 EXPORT_SYMBOL_GPL(hrtimer_cancel);
12671349 
....@@ -1357,16 +1439,18 @@
13571439 static void __hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
13581440 			   enum hrtimer_mode mode)
13591441 {
1360
-	bool softtimer;
1361
-	int base;
1442
+	bool softtimer = !!(mode & HRTIMER_MODE_SOFT);
13621443 	struct hrtimer_cpu_base *cpu_base;
1444
+	int base;
13631445 
1364
-	softtimer = !!(mode & HRTIMER_MODE_SOFT);
1365
-#ifdef CONFIG_PREEMPT_RT_FULL
1366
-	if (!softtimer && !(mode & HRTIMER_MODE_HARD))
1446
+	/*
1447
+	 * On PREEMPT_RT enabled kernels hrtimers which are not explicitely
1448
+	 * marked for hard interrupt expiry mode are moved into soft
1449
+	 * interrupt context for latency reasons and because the callbacks
1450
+	 * can invoke functions which might sleep on RT, e.g. spin_lock().
1451
+	 */
1452
+	if (IS_ENABLED(CONFIG_PREEMPT_RT) && !(mode & HRTIMER_MODE_HARD))
13671453 		softtimer = true;
1368
-#endif
1369
-	base = softtimer ? HRTIMER_MAX_CLOCK_BASES / 2 : 0;
13701454 
13711455 	memset(timer, 0, sizeof(struct hrtimer));
13721456 
....@@ -1380,8 +1464,10 @@
13801464 	if (clock_id == CLOCK_REALTIME && mode & HRTIMER_MODE_REL)
13811465 		clock_id = CLOCK_MONOTONIC;
13821466 
1467
+	base = softtimer ? HRTIMER_MAX_CLOCK_BASES / 2 : 0;
13831468 	base += hrtimer_clockid_to_base(clock_id);
13841469 	timer->is_soft = softtimer;
1470
+	timer->is_hard = !!(mode & HRTIMER_MODE_HARD);
13851471 	timer->base = &cpu_base->clock_base[base];
13861472 	timerqueue_init(&timer->node);
13871473 }
....@@ -1454,9 +1540,10 @@
14541540 static void __run_hrtimer(struct hrtimer_cpu_base *cpu_base,
14551541 			  struct hrtimer_clock_base *base,
14561542 			  struct hrtimer *timer, ktime_t *now,
1457
-			  unsigned long flags)
1543
+			  unsigned long flags) __must_hold(&cpu_base->lock)
14581544 {
14591545 	enum hrtimer_restart (*fn)(struct hrtimer *);
1546
+	bool expires_in_hardirq;
14601547 	int restart;
14611548 
14621549 	lockdep_assert_held(&cpu_base->lock);
....@@ -1491,7 +1578,11 @@
14911578 	 */
14921579 	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
14931580 	trace_hrtimer_expire_entry(timer, now);
1581
+	expires_in_hardirq = lockdep_hrtimer_enter(timer);
1582
+
14941583 	restart = fn(timer);
1584
+
1585
+	lockdep_hrtimer_exit(expires_in_hardirq);
14951586 	trace_hrtimer_expire_exit(timer);
14961587 	raw_spin_lock_irq(&cpu_base->lock);
14971588 
....@@ -1554,6 +1645,8 @@
15541645 				break;
15551646 
15561647 			__run_hrtimer(cpu_base, base, timer, &basenow, flags);
1648
+			if (active_mask == HRTIMER_ACTIVE_SOFT)
1649
+				hrtimer_sync_wait_running(cpu_base, flags);
15571650 		}
15581651 	}
15591652 }
....@@ -1564,7 +1657,7 @@
15641657 	unsigned long flags;
15651658 	ktime_t now;
15661659 
1567
-	spin_lock(&cpu_base->softirq_expiry_lock);
1660
+	hrtimer_cpu_base_lock_expiry(cpu_base);
15681661 	raw_spin_lock_irqsave(&cpu_base->lock, flags);
15691662 
15701663 	now = hrtimer_update_base(cpu_base);
....@@ -1574,7 +1667,7 @@
15741667 	hrtimer_update_softirq_timer(cpu_base, true);
15751668 
15761669 	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
1577
-	spin_unlock(&cpu_base->softirq_expiry_lock);
1670
+	hrtimer_cpu_base_unlock_expiry(cpu_base);
15781671 }
15791672 
15801673 #ifdef CONFIG_HIGH_RES_TIMERS
....@@ -1746,22 +1839,61 @@
17461839 	return HRTIMER_NORESTART;
17471840 }
17481841 
1749
-static void __hrtimer_init_sleeper(struct hrtimer_sleeper *sl,
1750
-				   clockid_t clock_id,
1751
-				   enum hrtimer_mode mode,
1752
-				   struct task_struct *task)
1842
+/**
1843
+ * hrtimer_sleeper_start_expires - Start a hrtimer sleeper timer
1844
+ * @sl:		sleeper to be started
1845
+ * @mode:	timer mode abs/rel
1846
+ *
1847
+ * Wrapper around hrtimer_start_expires() for hrtimer_sleeper based timers
1848
+ * to allow PREEMPT_RT to tweak the delivery mode (soft/hardirq context)
1849
+ */
1850
+void hrtimer_sleeper_start_expires(struct hrtimer_sleeper *sl,
1851
+				   enum hrtimer_mode mode)
17531852 {
1754
-#ifdef CONFIG_PREEMPT_RT_FULL
1755
-	if (!(mode & (HRTIMER_MODE_SOFT | HRTIMER_MODE_HARD))) {
1756
-		if (task_is_realtime(current) || system_state != SYSTEM_RUNNING)
1853
+	/*
1854
+	 * Make the enqueue delivery mode check work on RT. If the sleeper
1855
+	 * was initialized for hard interrupt delivery, force the mode bit.
1856
+	 * This is a special case for hrtimer_sleepers because
1857
+	 * hrtimer_init_sleeper() determines the delivery mode on RT so the
1858
+	 * fiddling with this decision is avoided at the call sites.
1859
+	 */
1860
+	if (IS_ENABLED(CONFIG_PREEMPT_RT) && sl->timer.is_hard)
1861
+		mode |= HRTIMER_MODE_HARD;
1862
+
1863
+	hrtimer_start_expires(&sl->timer, mode);
1864
+}
1865
+EXPORT_SYMBOL_GPL(hrtimer_sleeper_start_expires);
1866
+
1867
+static void __hrtimer_init_sleeper(struct hrtimer_sleeper *sl,
1868
+				   clockid_t clock_id, enum hrtimer_mode mode)
1869
+{
1870
+	/*
1871
+	 * On PREEMPT_RT enabled kernels hrtimers which are not explicitely
1872
+	 * marked for hard interrupt expiry mode are moved into soft
1873
+	 * interrupt context either for latency reasons or because the
1874
+	 * hrtimer callback takes regular spinlocks or invokes other
1875
+	 * functions which are not suitable for hard interrupt context on
1876
+	 * PREEMPT_RT.
1877
+	 *
1878
+	 * The hrtimer_sleeper callback is RT compatible in hard interrupt
1879
+	 * context, but there is a latency concern: Untrusted userspace can
1880
+	 * spawn many threads which arm timers for the same expiry time on
1881
+	 * the same CPU. That causes a latency spike due to the wakeup of
1882
+	 * a gazillion threads.
1883
+	 *
1884
+	 * OTOH, priviledged real-time user space applications rely on the
1885
+	 * low latency of hard interrupt wakeups. If the current task is in
1886
+	 * a real-time scheduling class, mark the mode for hard interrupt
1887
+	 * expiry.
1888
+	 */
1889
+	if (IS_ENABLED(CONFIG_PREEMPT_RT)) {
1890
+		if (task_is_realtime(current) && !(mode & HRTIMER_MODE_SOFT))
17571891 			mode |= HRTIMER_MODE_HARD;
1758
-		else
1759
-			mode |= HRTIMER_MODE_SOFT;
17601892 	}
1761
-#endif
1893
+
17621894 	__hrtimer_init(&sl->timer, clock_id, mode);
17631895 	sl->timer.function = hrtimer_wakeup;
1764
-	sl->task = task;
1896
+	sl->task = current;
17651897 }
17661898 
17671899 /**
....@@ -1769,35 +1901,22 @@
17691901  * @sl:		sleeper to be initialized
17701902  * @clock_id:	the clock to be used
17711903  * @mode:	timer mode abs/rel
1772
- * @task:	the task to wake up
17731904  */
17741905 void hrtimer_init_sleeper(struct hrtimer_sleeper *sl, clockid_t clock_id,
1775
-			  enum hrtimer_mode mode, struct task_struct *task)
1906
+			  enum hrtimer_mode mode)
17761907 {
17771908 	debug_init(&sl->timer, clock_id, mode);
1778
-	__hrtimer_init_sleeper(sl, clock_id, mode, task);
1909
+	__hrtimer_init_sleeper(sl, clock_id, mode);
17791910 
17801911 }
17811912 EXPORT_SYMBOL_GPL(hrtimer_init_sleeper);
1782
-
1783
-#ifdef CONFIG_DEBUG_OBJECTS_TIMERS
1784
-void hrtimer_init_sleeper_on_stack(struct hrtimer_sleeper *sl,
1785
-				   clockid_t clock_id,
1786
-				   enum hrtimer_mode mode,
1787
-				   struct task_struct *task)
1788
-{
1789
-	debug_object_init_on_stack(&sl->timer, &hrtimer_debug_descr);
1790
-	__hrtimer_init_sleeper(sl, clock_id, mode, task);
1791
-}
1792
-EXPORT_SYMBOL_GPL(hrtimer_init_sleeper_on_stack);
1793
-#endif
17941913 
17951914 int nanosleep_copyout(struct restart_block *restart, struct timespec64 *ts)
17961915 {
17971916 	switch(restart->nanosleep.type) {
17981917 #ifdef CONFIG_COMPAT_32BIT_TIME
17991918 	case TT_COMPAT:
1800
-		if (compat_put_timespec64(ts, restart->nanosleep.compat_rmtp))
1919
+		if (put_old_timespec32(ts, restart->nanosleep.compat_rmtp))
18011920 			return -EFAULT;
18021921 		break;
18031922 #endif
....@@ -1817,17 +1936,17 @@
18171936 
18181937 	do {
18191938 		set_current_state(TASK_INTERRUPTIBLE);
1820
-		hrtimer_start_expires(&t->timer, mode);
1939
+		hrtimer_sleeper_start_expires(t, mode);
18211940 
18221941 		if (likely(t->task))
18231942 			freezable_schedule();
18241943 
1825
-		__set_current_state(TASK_RUNNING);
18261944 		hrtimer_cancel(&t->timer);
18271945 		mode = HRTIMER_MODE_ABS;
18281946 
18291947 	} while (t->task && !signal_pending(current));
18301948 
1949
+	__set_current_state(TASK_RUNNING);
18311950 
18321951 	if (!t->task)
18331952 		return 0;
....@@ -1852,15 +1971,15 @@
18521971 	int ret;
18531972 
18541973 	hrtimer_init_sleeper_on_stack(&t, restart->nanosleep.clockid,
1855
-				      HRTIMER_MODE_ABS, current);
1974
+				      HRTIMER_MODE_ABS);
18561975 	hrtimer_set_expires_tv64(&t.timer, restart->nanosleep.expires);
18571976 	ret = do_nanosleep(&t, HRTIMER_MODE_ABS);
18581977 	destroy_hrtimer_on_stack(&t.timer);
18591978 	return ret;
18601979 }
18611980 
1862
-long hrtimer_nanosleep(const struct timespec64 *rqtp,
1863
-		       const enum hrtimer_mode mode, const clockid_t clockid)
1981
+long hrtimer_nanosleep(ktime_t rqtp, const enum hrtimer_mode mode,
1982
+		       const clockid_t clockid)
18641983 {
18651984 	struct restart_block *restart;
18661985 	struct hrtimer_sleeper t;
....@@ -1871,8 +1990,8 @@
18711990 	if (dl_task(current) || rt_task(current))
18721991 		slack = 0;
18731992 
1874
-	hrtimer_init_sleeper_on_stack(&t, clockid, mode, current);
1875
-	hrtimer_set_expires_range_ns(&t.timer, timespec64_to_ktime(*rqtp), slack);
1993
+	hrtimer_init_sleeper_on_stack(&t, clockid, mode);
1994
+	hrtimer_set_expires_range_ns(&t.timer, rqtp, slack);
18761995 	ret = do_nanosleep(&t, mode);
18771996 	if (ret != -ERESTART_RESTARTBLOCK)
18781997 		goto out;
....@@ -1892,7 +2011,7 @@
18922011 	return ret;
18932012 }
18942013 
1895
-#if !defined(CONFIG_64BIT_TIME) || defined(CONFIG_64BIT)
2014
+#ifdef CONFIG_64BIT
18962015 
18972016 SYSCALL_DEFINE2(nanosleep, struct __kernel_timespec __user *, rqtp,
18982017 		struct __kernel_timespec __user *, rmtp)
....@@ -1905,62 +2024,34 @@
19052024 	if (!timespec64_valid(&tu))
19062025 		return -EINVAL;
19072026 
2027
+	current->restart_block.fn = do_no_restart_syscall;
19082028 	current->restart_block.nanosleep.type = rmtp ? TT_NATIVE : TT_NONE;
19092029 	current->restart_block.nanosleep.rmtp = rmtp;
1910
-	return hrtimer_nanosleep(&tu, HRTIMER_MODE_REL, CLOCK_MONOTONIC);
2030
+	return hrtimer_nanosleep(timespec64_to_ktime(tu), HRTIMER_MODE_REL,
2031
+				 CLOCK_MONOTONIC);
19112032 }
19122033 
19132034 #endif
19142035 
19152036 #ifdef CONFIG_COMPAT_32BIT_TIME
19162037 
1917
-COMPAT_SYSCALL_DEFINE2(nanosleep, struct compat_timespec __user *, rqtp,
1918
-		       struct compat_timespec __user *, rmtp)
2038
+SYSCALL_DEFINE2(nanosleep_time32, struct old_timespec32 __user *, rqtp,
2039
+		       struct old_timespec32 __user *, rmtp)
19192040 {
19202041 	struct timespec64 tu;
19212042 
1922
-	if (compat_get_timespec64(&tu, rqtp))
2043
+	if (get_old_timespec32(&tu, rqtp))
19232044 		return -EFAULT;
19242045 
19252046 	if (!timespec64_valid(&tu))
19262047 		return -EINVAL;
19272048 
2049
+	current->restart_block.fn = do_no_restart_syscall;
19282050 	current->restart_block.nanosleep.type = rmtp ? TT_COMPAT : TT_NONE;
19292051 	current->restart_block.nanosleep.compat_rmtp = rmtp;
1930
-	return hrtimer_nanosleep(&tu, HRTIMER_MODE_REL, CLOCK_MONOTONIC);
2052
+	return hrtimer_nanosleep(timespec64_to_ktime(tu), HRTIMER_MODE_REL,
2053
+				 CLOCK_MONOTONIC);
19312054 }
1932
-#endif
1933
-
1934
-#ifdef CONFIG_PREEMPT_RT_FULL
1935
-/*
1936
- * Sleep for 1 ms in hope whoever holds what we want will let it go.
1937
- */
1938
-void cpu_chill(void)
1939
-{
1940
-	unsigned int freeze_flag = current->flags & PF_NOFREEZE;
1941
-	struct task_struct *self = current;
1942
-	ktime_t chill_time;
1943
-
1944
-	raw_spin_lock_irq(&self->pi_lock);
1945
-	self->saved_state = self->state;
1946
-	__set_current_state_no_track(TASK_UNINTERRUPTIBLE);
1947
-	raw_spin_unlock_irq(&self->pi_lock);
1948
-
1949
-	chill_time = ktime_set(0, NSEC_PER_MSEC);
1950
-
1951
-	current->flags |= PF_NOFREEZE;
1952
-	sleeping_lock_inc();
1953
-	schedule_hrtimeout(&chill_time, HRTIMER_MODE_REL_HARD);
1954
-	sleeping_lock_dec();
1955
-	if (!freeze_flag)
1956
-		current->flags &= ~PF_NOFREEZE;
1957
-
1958
-	raw_spin_lock_irq(&self->pi_lock);
1959
-	__set_current_state_no_track(self->saved_state);
1960
-	self->saved_state = TASK_RUNNING;
1961
-	raw_spin_unlock_irq(&self->pi_lock);
1962
-}
1963
-EXPORT_SYMBOL(cpu_chill);
19642055 #endif
19652056 
19662057 /*
....@@ -1972,8 +2063,11 @@
19722063 	int i;
19732064 
19742065 	for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) {
1975
-		cpu_base->clock_base[i].cpu_base = cpu_base;
1976
-		timerqueue_init_head(&cpu_base->clock_base[i].active);
2066
+		struct hrtimer_clock_base *clock_b = &cpu_base->clock_base[i];
2067
+
2068
+		clock_b->cpu_base = cpu_base;
2069
+		seqcount_raw_spinlock_init(&clock_b->seq, &cpu_base->lock);
2070
+		timerqueue_init_head(&clock_b->active);
19772071 	}
19782072 
19792073 	cpu_base->cpu = cpu;
....@@ -1984,7 +2078,7 @@
19842078 	cpu_base->softirq_next_timer = NULL;
19852079 	cpu_base->expires_next = KTIME_MAX;
19862080 	cpu_base->softirq_expires_next = KTIME_MAX;
1987
-	spin_lock_init(&cpu_base->softirq_expiry_lock);
2081
+	hrtimer_cpu_base_init_expiry_lock(cpu_base);
19882082 	return 0;
19892083 }
19902084 
....@@ -2103,10 +2197,9 @@
21032197 		return -EINTR;
21042198 	}
21052199 
2106
-	hrtimer_init_sleeper_on_stack(&t, clock_id, mode, current);
2200
+	hrtimer_init_sleeper_on_stack(&t, clock_id, mode);
21072201 	hrtimer_set_expires_range_ns(&t.timer, *expires, delta);
2108
-
2109
-	hrtimer_start_expires(&t.timer, mode);
2202
+	hrtimer_sleeper_start_expires(&t, mode);
21102203 
21112204 	if (likely(t.task))
21122205 		schedule();
....@@ -2118,6 +2211,7 @@
21182211 
21192212 	return !t.task ? 0 : -EINTR;
21202213 }
2214
+EXPORT_SYMBOL_GPL(schedule_hrtimeout_range_clock);
21212215 
21222216 /**
21232217  * schedule_hrtimeout_range - sleep until timeout