forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2023-12-08 01573e231f18eb2d99162747186f59511f56b64d 
 kernel/drivers/clocksource/arm_arch_timer.c
....@@ -1,15 +1,12 @@
1
+// SPDX-License-Identifier: GPL-2.0-only
12 /*
23  *  linux/drivers/clocksource/arm_arch_timer.c
34  *
45  *  Copyright (C) 2011 ARM Ltd.
56  *  All Rights Reserved
6
- *
7
- * This program is free software; you can redistribute it and/or modify
8
- * it under the terms of the GNU General Public License version 2 as
9
- * published by the Free Software Foundation.
107  */
118 
12
-#define pr_fmt(fmt)	"arm_arch_timer: " fmt
9
+#define pr_fmt(fmt) 	"arch_timer: " fmt
1310 
1411 #include <linux/init.h>
1512 #include <linux/kernel.h>
....@@ -33,9 +30,6 @@
3330 
3431 #include <clocksource/arm_arch_timer.h>
3532 
36
-#undef pr_fmt
37
-#define pr_fmt(fmt) "arch_timer: " fmt
38
-
3933 #define CNTTIDR		0x08
4034 #define CNTTIDR_VIRT(n)	(BIT(1) << ((n) * 4))
4135 
....@@ -52,8 +46,6 @@
5246 #define CNTFRQ		0x10
5347 #define CNTP_TVAL	0x28
5448 #define CNTP_CTL	0x2c
55
-#define CNTCVAL_LO	0x30
56
-#define CNTCVAL_HI	0x34
5749 #define CNTV_TVAL	0x38
5850 #define CNTV_CTL	0x3c
5951 
....@@ -77,7 +69,11 @@
7769 static bool arch_timer_c3stop;
7870 static bool arch_timer_mem_use_virtual;
7971 static bool arch_counter_suspend_stop;
80
-static bool vdso_default = true;
72
+#ifdef CONFIG_GENERIC_GETTIMEOFDAY
73
+static enum vdso_clock_mode vdso_default = VDSO_CLOCKMODE_ARCHTIMER;
74
+#else
75
+static enum vdso_clock_mode vdso_default = VDSO_CLOCKMODE_NONE;
76
+#endif /* CONFIG_GENERIC_GETTIMEOFDAY */
8177 
8278 static cpumask_t evtstrm_available = CPU_MASK_NONE;
8379 static bool evtstrm_enable = IS_ENABLED(CONFIG_ARM_ARCH_TIMER_EVTSTREAM);
....@@ -152,6 +148,26 @@
152148 	}
153149 
154150 	return val;
151
+}
152
+
153
+static notrace u64 arch_counter_get_cntpct_stable(void)
154
+{
155
+	return __arch_counter_get_cntpct_stable();
156
+}
157
+
158
+static notrace u64 arch_counter_get_cntpct(void)
159
+{
160
+	return __arch_counter_get_cntpct();
161
+}
162
+
163
+static notrace u64 arch_counter_get_cntvct_stable(void)
164
+{
165
+	return __arch_counter_get_cntvct_stable();
166
+}
167
+
168
+static notrace u64 arch_counter_get_cntvct(void)
169
+{
170
+	return __arch_counter_get_cntvct();
155171 }
156172 
157173 /*
....@@ -367,8 +383,7 @@
367383 DEFINE_PER_CPU(const struct arch_timer_erratum_workaround *, timer_unstable_counter_workaround);
368384 EXPORT_SYMBOL_GPL(timer_unstable_counter_workaround);
369385 
370
-DEFINE_STATIC_KEY_FALSE(arch_timer_read_ool_enabled);
371
-EXPORT_SYMBOL_GPL(arch_timer_read_ool_enabled);
386
+static atomic_t timer_unstable_counter_workaround_in_use = ATOMIC_INIT(0);
372387 
373388 static void erratum_set_next_event_tval_generic(const int access, unsigned long evt,
374389 						struct clock_event_device *clk)
....@@ -381,10 +396,10 @@
381396 	ctrl &= ~ARCH_TIMER_CTRL_IT_MASK;
382397 
383398 	if (access == ARCH_TIMER_PHYS_ACCESS) {
384
-		cval = evt + arch_counter_get_cntpct();
399
+		cval = evt + arch_counter_get_cntpct_stable();
385400 		write_sysreg(cval, cntp_cval_el0);
386401 	} else {
387
-		cval = evt + arch_counter_get_cntvct();
402
+		cval = evt + arch_counter_get_cntvct_stable();
388403 		write_sysreg(cval, cntv_cval_el0);
389404 	}
390405 
....@@ -465,6 +480,14 @@
465480 		.set_next_event_virt = erratum_set_next_event_tval_virt,
466481 	},
467482 #endif
483
+#ifdef CONFIG_ARM64_ERRATUM_1418040
484
+	{
485
+		.match_type = ate_match_local_cap_id,
486
+		.id = (void *)ARM64_WORKAROUND_1418040,
487
+		.desc = "ARM erratum 1418040",
488
+		.disable_compat_vdso = true,
489
+	},
490
+#endif
468491 };
469492 
470493 typedef bool (*ate_match_fn_t)(const struct arch_timer_erratum_workaround *,
....@@ -539,11 +562,8 @@
539562 			per_cpu(timer_unstable_counter_workaround, i) = wa;
540563 	}
541564 
542
-	/*
543
-	 * Use the locked version, as we're called from the CPU
544
-	 * hotplug framework. Otherwise, we end-up in deadlock-land.
545
-	 */
546
-	static_branch_enable_cpuslocked(&arch_timer_read_ool_enabled);
565
+	if (wa->read_cntvct_el0 || wa->read_cntpct_el0)
566
+		atomic_set(&timer_unstable_counter_workaround_in_use, 1);
547567 
548568 	/*
549569 	 * Don't use the vdso fastpath if errata require using the
....@@ -552,15 +572,18 @@
552572 	 * change both the default value and the vdso itself.
553573 	 */
554574 	if (wa->read_cntvct_el0) {
555
-		clocksource_counter.archdata.vdso_direct = false;
556
-		vdso_default = false;
575
+		clocksource_counter.vdso_clock_mode = VDSO_CLOCKMODE_NONE;
576
+		vdso_default = VDSO_CLOCKMODE_NONE;
577
+	} else if (wa->disable_compat_vdso && vdso_default != VDSO_CLOCKMODE_NONE) {
578
+		vdso_default = VDSO_CLOCKMODE_ARCHTIMER_NOCOMPAT;
579
+		clocksource_counter.vdso_clock_mode = vdso_default;
557580 	}
558581 }
559582 
560583 static void arch_timer_check_ool_workaround(enum arch_timer_erratum_match_type type,
561584 					    void *arg)
562585 {
563
-	const struct arch_timer_erratum_workaround *wa;
586
+	const struct arch_timer_erratum_workaround *wa, *__wa;
564587 	ate_match_fn_t match_fn = NULL;
565588 	bool local = false;
566589 
....@@ -584,53 +607,32 @@
584607 	if (!wa)
585608 		return;
586609 
587
-	if (needs_unstable_timer_counter_workaround()) {
588
-		const struct arch_timer_erratum_workaround *__wa;
589
-		__wa = __this_cpu_read(timer_unstable_counter_workaround);
590
-		if (__wa && wa != __wa)
591
-			pr_warn("Can't enable workaround for %s (clashes with %s\n)",
592
-				wa->desc, __wa->desc);
610
+	__wa = __this_cpu_read(timer_unstable_counter_workaround);
611
+	if (__wa && wa != __wa)
612
+		pr_warn("Can't enable workaround for %s (clashes with %s\n)",
613
+			wa->desc, __wa->desc);
593614 
594
-		if (__wa)
595
-			return;
596
-	}
615
+	if (__wa)
616
+		return;
597617 
598618 	arch_timer_enable_workaround(wa, local);
599619 	pr_info("Enabling %s workaround for %s\n",
600620 		local ? "local" : "global", wa->desc);
601621 }
602622 
603
-#define erratum_handler(fn, r, ...)					\
604
-({									\
605
-	bool __val;							\
606
-	if (needs_unstable_timer_counter_workaround()) {		\
607
-		const struct arch_timer_erratum_workaround *__wa;	\
608
-		__wa = __this_cpu_read(timer_unstable_counter_workaround); \
609
-		if (__wa && __wa->fn) {					\
610
-			r = __wa->fn(__VA_ARGS__);			\
611
-			__val = true;					\
612
-		} else {						\
613
-			__val = false;					\
614
-		}							\
615
-	} else {							\
616
-		__val = false;						\
617
-	}								\
618
-	__val;								\
619
-})
620
-
621623 static bool arch_timer_this_cpu_has_cntvct_wa(void)
622624 {
623
-	const struct arch_timer_erratum_workaround *wa;
625
+	return has_erratum_handler(read_cntvct_el0);
626
+}
624627 
625
-	wa = __this_cpu_read(timer_unstable_counter_workaround);
626
-	return wa && wa->read_cntvct_el0;
628
+static bool arch_timer_counter_has_wa(void)
629
+{
630
+	return atomic_read(&timer_unstable_counter_workaround_in_use);
627631 }
628632 #else
629633 #define arch_timer_check_ool_workaround(t,a)		do { } while(0)
630
-#define erratum_set_next_event_tval_virt(...)		({BUG(); 0;})
631
-#define erratum_set_next_event_tval_phys(...)		({BUG(); 0;})
632
-#define erratum_handler(fn, r, ...)			({false;})
633634 #define arch_timer_this_cpu_has_cntvct_wa()		({false;})
635
+#define arch_timer_counter_has_wa()			({false;})
634636 #endif /* CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND */
635637 
636638 static __always_inline irqreturn_t timer_handler(const int access,
....@@ -723,11 +725,6 @@
723725 static int arch_timer_set_next_event_virt(unsigned long evt,
724726 					  struct clock_event_device *clk)
725727 {
726
-	int ret;
727
-
728
-	if (erratum_handler(set_next_event_virt, ret, evt, clk))
729
-		return ret;
730
-
731728 	set_next_event(ARCH_TIMER_VIRT_ACCESS, evt, clk);
732729 	return 0;
733730 }
....@@ -735,11 +732,6 @@
735732 static int arch_timer_set_next_event_phys(unsigned long evt,
736733 					  struct clock_event_device *clk)
737734 {
738
-	int ret;
739
-
740
-	if (erratum_handler(set_next_event_phys, ret, evt, clk))
741
-		return ret;
742
-
743735 	set_next_event(ARCH_TIMER_PHYS_ACCESS, evt, clk);
744736 	return 0;
745737 }
....@@ -764,6 +756,10 @@
764756 	clk->features = CLOCK_EVT_FEAT_ONESHOT;
765757 
766758 	if (type == ARCH_TIMER_TYPE_CP15) {
759
+		typeof(clk->set_next_event) sne;
760
+
761
+		arch_timer_check_ool_workaround(ate_match_local_cap_id, NULL);
762
+
767763 		if (arch_timer_c3stop)
768764 			clk->features |= CLOCK_EVT_FEAT_C3STOP;
769765 		clk->name = "arch_sys_timer";
....@@ -774,20 +770,20 @@
774770 		case ARCH_TIMER_VIRT_PPI:
775771 			clk->set_state_shutdown = arch_timer_shutdown_virt;
776772 			clk->set_state_oneshot_stopped = arch_timer_shutdown_virt;
777
-			clk->set_next_event = arch_timer_set_next_event_virt;
773
+			sne = erratum_handler(set_next_event_virt);
778774 			break;
779775 		case ARCH_TIMER_PHYS_SECURE_PPI:
780776 		case ARCH_TIMER_PHYS_NONSECURE_PPI:
781777 		case ARCH_TIMER_HYP_PPI:
782778 			clk->set_state_shutdown = arch_timer_shutdown_phys;
783779 			clk->set_state_oneshot_stopped = arch_timer_shutdown_phys;
784
-			clk->set_next_event = arch_timer_set_next_event_phys;
780
+			sne = erratum_handler(set_next_event_phys);
785781 			break;
786782 		default:
787783 			BUG();
788784 		}
789785 
790
-		arch_timer_check_ool_workaround(ate_match_local_cap_id, NULL);
786
+		clk->set_next_event = sne;
791787 	} else {
792788 		clk->features |= CLOCK_EVT_FEAT_DYNIRQ;
793789 		clk->name = "arch_mem_timer";
....@@ -820,10 +816,7 @@
820816 	cntkctl |= (divider << ARCH_TIMER_EVT_TRIGGER_SHIFT)
821817 			| ARCH_TIMER_VIRT_EVT_EN;
822818 	arch_timer_set_cntkctl(cntkctl);
823
-	elf_hwcap |= HWCAP_EVTSTRM;
824
-#ifdef CONFIG_COMPAT
825
-	compat_elf_hwcap |= COMPAT_HWCAP_EVTSTRM;
826
-#endif
819
+	arch_timer_set_evtstrm_feature();
827820 	cpumask_set_cpu(smp_processor_id(), &evtstrm_available);
828821 }
829822 
....@@ -916,6 +909,17 @@
916909 	return 0;
917910 }
918911 
912
+static int validate_timer_rate(void)
913
+{
914
+	if (!arch_timer_rate)
915
+		return -EINVAL;
916
+
917
+	/* Arch timer frequency < 1MHz can cause trouble */
918
+	WARN_ON(arch_timer_rate < 1000000);
919
+
920
+	return 0;
921
+}
922
+
919923 /*
920924  * For historical reasons, when probing with DT we use whichever (non-zero)
921925  * rate was probed first, and don't verify that others match. If the first node
....@@ -931,7 +935,7 @@
931935 		arch_timer_rate = rate;
932936 
933937 	/* Check the timer frequency. */
934
-	if (arch_timer_rate == 0)
938
+	if (validate_timer_rate())
935939 		pr_warn("frequency not available\n");
936940 }
937941 
....@@ -957,7 +961,6 @@
957961 {
958962 	return arch_timer_rate;
959963 }
960
-EXPORT_SYMBOL_GPL(arch_timer_get_rate);
961964 
962965 bool arch_timer_evtstrm_available(void)
963966 {
....@@ -968,24 +971,6 @@
968971 	 */
969972 	return cpumask_test_cpu(raw_smp_processor_id(), &evtstrm_available);
970973 }
971
-
972
-void arch_timer_mem_get_cval(u32 *lo, u32 *hi)
973
-{
974
-	u32 ctrl;
975
-
976
-	*lo = *hi = ~0U;
977
-
978
-	if (!arch_counter_base)
979
-		return;
980
-
981
-	ctrl = readl_relaxed(arch_counter_base + CNTV_CTL);
982
-
983
-	if (ctrl & ARCH_TIMER_CTRL_ENABLE) {
984
-		*lo = readl_relaxed(arch_counter_base + CNTCVAL_LO);
985
-		*hi = readl_relaxed(arch_counter_base + CNTCVAL_HI);
986
-	}
987
-}
988
-EXPORT_SYMBOL_GPL(arch_timer_mem_get_cval);
989974 
990975 static u64 arch_counter_get_cntvct_mem(void)
991976 {
....@@ -1013,13 +998,23 @@
1013998 
1014999 	/* Register the CP15 based counter if we have one */
10151000 	if (type & ARCH_TIMER_TYPE_CP15) {
1016
-		if ((IS_ENABLED(CONFIG_ARM64) && !is_hyp_mode_available()) ||
1017
-		    arch_timer_uses_ppi == ARCH_TIMER_VIRT_PPI)
1018
-			arch_timer_read_counter = arch_counter_get_cntvct;
1019
-		else
1020
-			arch_timer_read_counter = arch_counter_get_cntpct;
1001
+		u64 (*rd)(void);
10211002 
1022
-		clocksource_counter.archdata.vdso_direct = vdso_default;
1003
+		if ((IS_ENABLED(CONFIG_ARM64) && !is_hyp_mode_available()) ||
1004
+		    arch_timer_uses_ppi == ARCH_TIMER_VIRT_PPI) {
1005
+			if (arch_timer_counter_has_wa())
1006
+				rd = arch_counter_get_cntvct_stable;
1007
+			else
1008
+				rd = arch_counter_get_cntvct;
1009
+		} else {
1010
+			if (arch_timer_counter_has_wa())
1011
+				rd = arch_counter_get_cntpct_stable;
1012
+			else
1013
+				rd = arch_counter_get_cntpct;
1014
+		}
1015
+
1016
+		arch_timer_read_counter = rd;
1017
+		clocksource_counter.vdso_clock_mode = vdso_default;
10231018 	} else {
10241019 		arch_timer_read_counter = arch_counter_get_cntvct_mem;
10251020 	}
....@@ -1070,7 +1065,7 @@
10701065 	} else if (action == CPU_PM_ENTER_FAILED || action == CPU_PM_EXIT) {
10711066 		arch_timer_set_cntkctl(__this_cpu_read(saved_cntkctl));
10721067 
1073
-		if (elf_hwcap & HWCAP_EVTSTRM)
1068
+		if (arch_timer_have_evtstrm_feature())
10741069 			cpumask_set_cpu(smp_processor_id(), &evtstrm_available);
10751070 	}
10761071 	return NOTIFY_OK;
....@@ -1276,6 +1271,13 @@
12761271 	return ARCH_TIMER_PHYS_SECURE_PPI;
12771272 }
12781273 
1274
+static void __init arch_timer_populate_kvm_info(void)
1275
+{
1276
+	arch_timer_kvm_info.virtual_irq = arch_timer_ppi[ARCH_TIMER_VIRT_PPI];
1277
+	if (is_kernel_in_hyp_mode())
1278
+		arch_timer_kvm_info.physical_irq = arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI];
1279
+}
1280
+
12791281 static int __init arch_timer_of_init(struct device_node *np)
12801282 {
12811283 	int i, ret;
....@@ -1290,7 +1292,7 @@
12901292 	for (i = ARCH_TIMER_PHYS_SECURE_PPI; i < ARCH_TIMER_MAX_TIMER_PPI; i++)
12911293 		arch_timer_ppi[i] = irq_of_parse_and_map(np, i);
12921294 
1293
-	arch_timer_kvm_info.virtual_irq = arch_timer_ppi[ARCH_TIMER_VIRT_PPI];
1295
+	arch_timer_populate_kvm_info();
12941296 
12951297 	rate = arch_timer_get_cntfrq();
12961298 	arch_timer_of_configure_rate(rate, np);
....@@ -1606,10 +1608,8 @@
16061608 	arch_timers_present |= ARCH_TIMER_TYPE_CP15;
16071609 
16081610 	ret = acpi_gtdt_init(table, &platform_timer_count);
1609
-	if (ret) {
1610
-		pr_err("Failed to init GTDT table.\n");
1611
+	if (ret)
16111612 		return ret;
1612
-	}
16131613 
16141614 	arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI] =
16151615 		acpi_gtdt_map_ppi(ARCH_TIMER_PHYS_NONSECURE_PPI);
....@@ -1620,16 +1620,17 @@
16201620 	arch_timer_ppi[ARCH_TIMER_HYP_PPI] =
16211621 		acpi_gtdt_map_ppi(ARCH_TIMER_HYP_PPI);
16221622 
1623
-	arch_timer_kvm_info.virtual_irq = arch_timer_ppi[ARCH_TIMER_VIRT_PPI];
1623
+	arch_timer_populate_kvm_info();
16241624 
16251625 	/*
16261626 	 * When probing via ACPI, we have no mechanism to override the sysreg
16271627 	 * CNTFRQ value. This *must* be correct.
16281628 	 */
16291629 	arch_timer_rate = arch_timer_get_cntfrq();
1630
-	if (!arch_timer_rate) {
1630
+	ret = validate_timer_rate();
1631
+	if (ret) {
16311632 		pr_err(FW_BUG "frequency not available.\n");
1632
-		return -EINVAL;
1633
+		return ret;
16331634 	}
16341635 
16351636 	arch_timer_uses_ppi = arch_timer_select_ppi();