forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2024-10-12 a5969cabbb4660eab42b6ef0412cbbd1200cf14d 
 kernel/kernel/time/sched_clock.c
....@@ -1,10 +1,7 @@
1
+// SPDX-License-Identifier: GPL-2.0
12 /*
2
- * sched_clock.c: Generic sched_clock() support, to extend low level
3
- *                hardware time counters to full 64-bit ns values.
4
- *
5
- * This program is free software; you can redistribute it and/or modify
6
- * it under the terms of the GNU General Public License version 2 as
7
- * published by the Free Software Foundation.
3
+ * Generic sched_clock() support, to extend low level hardware time
4
+ * counters to full 64-bit ns values.
85  */
96 #include <linux/clocksource.h>
107 #include <linux/init.h>
....@@ -19,31 +16,9 @@
1916 #include <linux/sched_clock.h>
2017 #include <linux/seqlock.h>
2118 #include <linux/bitops.h>
19
+#include <trace/hooks/epoch.h>
2220 
23
-/**
24
- * struct clock_read_data - data required to read from sched_clock()
25
- *
26
- * @epoch_ns:		sched_clock() value at last update
27
- * @epoch_cyc:		Clock cycle value at last update.
28
- * @sched_clock_mask:   Bitmask for two's complement subtraction of non 64bit
29
- *			clocks.
30
- * @read_sched_clock:	Current clock source (or dummy source when suspended).
31
- * @mult:		Multipler for scaled math conversion.
32
- * @shift:		Shift value for scaled math conversion.
33
- *
34
- * Care must be taken when updating this structure; it is read by
35
- * some very hot code paths. It occupies <=40 bytes and, when combined
36
- * with the seqcount used to synchronize access, comfortably fits into
37
- * a 64 byte cache line.
38
- */
39
-struct clock_read_data {
40
-	u64 epoch_ns;
41
-	u64 epoch_cyc;
42
-	u64 sched_clock_mask;
43
-	u64 (*read_sched_clock)(void);
44
-	u32 mult;
45
-	u32 shift;
46
-};
21
+#include "timekeeping.h"
4722 
4823 /**
4924  * struct clock_data - all data needed for sched_clock() (including
....@@ -61,7 +36,7 @@
6136  * into a single 64-byte cache line.
6237  */
6338 struct clock_data {
64
-	seqcount_t		seq;
39
+	seqcount_latch_t	seq;
6540 	struct clock_read_data	read_data[2];
6641 	ktime_t			wrap_kt;
6742 	unsigned long		rate;
....@@ -94,20 +69,30 @@
9469 	return (cyc * mult) >> shift;
9570 }
9671 
72
+notrace struct clock_read_data *sched_clock_read_begin(unsigned int *seq)
73
+{
74
+	*seq = raw_read_seqcount_latch(&cd.seq);
75
+	return cd.read_data + (*seq & 1);
76
+}
77
+
78
+notrace int sched_clock_read_retry(unsigned int seq)
79
+{
80
+	return read_seqcount_latch_retry(&cd.seq, seq);
81
+}
82
+
9783 unsigned long long notrace sched_clock(void)
9884 {
9985 	u64 cyc, res;
100
-	unsigned long seq;
86
+	unsigned int seq;
10187 	struct clock_read_data *rd;
10288 
10389 	do {
104
-		seq = raw_read_seqcount(&cd.seq);
105
-		rd = cd.read_data + (seq & 1);
90
+		rd = sched_clock_read_begin(&seq);
10691 
10792 		cyc = (rd->read_sched_clock() - rd->epoch_cyc) &
10893 		      rd->sched_clock_mask;
10994 		res = rd->epoch_ns + cyc_to_ns(cyc, rd->mult, rd->shift);
110
-	} while (read_seqcount_retry(&cd.seq, seq));
95
+	} while (sched_clock_read_retry(seq));
11196 
11297 	return res;
11398 }
....@@ -165,19 +150,19 @@
165150 	return HRTIMER_RESTART;
166151 }
167152 
168
-void __init
169
-sched_clock_register(u64 (*read)(void), int bits, unsigned long rate)
153
+void sched_clock_register(u64 (*read)(void), int bits, unsigned long rate)
170154 {
171155 	u64 res, wrap, new_mask, new_epoch, cyc, ns;
172156 	u32 new_mult, new_shift;
173
-	unsigned long r;
157
+	unsigned long r, flags;
174158 	char r_unit;
175159 	struct clock_read_data rd;
176160 
177161 	if (cd.rate > rate)
178162 		return;
179163 
180
-	WARN_ON(!irqs_disabled());
164
+	/* Cannot register a sched_clock with interrupts on */
165
+	local_irq_save(flags);
181166 
182167 	/* Calculate the mult/shift to convert counter ticks to ns. */
183168 	clocks_calc_mult_shift(&new_mult, &new_shift, rate, NSEC_PER_SEC, 3600);
....@@ -208,7 +193,8 @@
208193 
209194 	if (sched_clock_timer.function != NULL) {
210195 		/* update timeout for clock wrap */
211
-		hrtimer_start(&sched_clock_timer, cd.wrap_kt, HRTIMER_MODE_REL);
196
+		hrtimer_start(&sched_clock_timer, cd.wrap_kt,
197
+			      HRTIMER_MODE_REL_HARD);
212198 	}
213199 
214200 	r = rate;
....@@ -234,14 +220,17 @@
234220 	if (irqtime > 0 || (irqtime == -1 && rate >= 1000000))
235221 		enable_sched_clock_irqtime();
236222 
237
-	pr_debug("Registered %pF as sched_clock source\n", read);
223
+	local_irq_restore(flags);
224
+
225
+	pr_debug("Registered %pS as sched_clock source\n", read);
238226 }
227
+EXPORT_SYMBOL_GPL(sched_clock_register);
239228 
240229 void __init generic_sched_clock_init(void)
241230 {
242231 	/*
243232 	 * If no sched_clock() function has been provided at that point,
244
-	 * make it the final one one.
233
+	 * make it the final one.
245234 	 */
246235 	if (cd.actual_read_sched_clock == jiffy_sched_clock_read)
247236 		sched_clock_register(jiffy_sched_clock_read, BITS_PER_LONG, HZ);
....@@ -252,9 +241,9 @@
252241 	 * Start the timer to keep sched_clock() properly updated and
253242 	 * sets the initial epoch.
254243 	 */
255
-	hrtimer_init(&sched_clock_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
244
+	hrtimer_init(&sched_clock_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD);
256245 	sched_clock_timer.function = sched_clock_poll;
257
-	hrtimer_start(&sched_clock_timer, cd.wrap_kt, HRTIMER_MODE_REL);
246
+	hrtimer_start(&sched_clock_timer, cd.wrap_kt, HRTIMER_MODE_REL_HARD);
258247 }
259248 
260249 /*
....@@ -270,7 +259,7 @@
270259  */
271260 static u64 notrace suspended_sched_clock_read(void)
272261 {
273
-	unsigned long seq = raw_read_seqcount(&cd.seq);
262
+	unsigned int seq = raw_read_seqcount_latch(&cd.seq);
274263 
275264 	return cd.read_data[seq & 1].epoch_cyc;
276265 }
....@@ -282,6 +271,7 @@
282271 	update_sched_clock();
283272 	hrtimer_cancel(&sched_clock_timer);
284273 	rd->read_sched_clock = suspended_sched_clock_read;
274
+	trace_android_vh_show_suspend_epoch_val(rd->epoch_ns, rd->epoch_cyc);
285275 
286276 	return 0;
287277 }
....@@ -291,8 +281,9 @@
291281 	struct clock_read_data *rd = &cd.read_data[0];
292282 
293283 	rd->epoch_cyc = cd.actual_read_sched_clock();
294
-	hrtimer_start(&sched_clock_timer, cd.wrap_kt, HRTIMER_MODE_REL);
284
+	hrtimer_start(&sched_clock_timer, cd.wrap_kt, HRTIMER_MODE_REL_HARD);
295285 	rd->read_sched_clock = cd.actual_read_sched_clock;
286
+	trace_android_vh_show_resume_epoch_val(rd->epoch_cyc);
296287 }
297288 
298289 static struct syscore_ops sched_clock_ops = {