forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2023-12-11 6778948f9de86c3cfaf36725a7c87dcff9ba247f 
 kernel/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_driver.c
....@@ -1,7 +1,7 @@
11 // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
22 /*
33  *
4
- * (C) COPYRIGHT 2010-2021 ARM Limited. All rights reserved.
4
+ * (C) COPYRIGHT 2010-2022 ARM Limited. All rights reserved.
55  *
66  * This program is free software and is provided to you under the terms of the
77  * GNU General Public License version 2 as published by the Free Software
....@@ -39,7 +39,8 @@
3939 
4040 #include <mali_kbase_reset_gpu.h>
4141 #include <mali_kbase_ctx_sched.h>
42
-#include <mali_kbase_hwcnt_context.h>
42
+#include <hwcnt/mali_kbase_hwcnt_context.h>
43
+#include <mali_kbase_pbha.h>
4344 #include <backend/gpu/mali_kbase_cache_policy_backend.h>
4445 #include <device/mali_kbase_device.h>
4546 #include <backend/gpu/mali_kbase_irq_internal.h>
....@@ -51,6 +52,10 @@
5152 #endif /* CONFIG_MALI_ARBITER_SUPPORT */
5253 #if MALI_USE_CSF
5354 #include <csf/ipa_control/mali_kbase_csf_ipa_control.h>
55
+#endif
56
+
57
+#if MALI_USE_CSF
58
+#include <linux/delay.h>
5459 #endif
5560 
5661 #include <linux/of.h>
....@@ -71,16 +76,16 @@
7176 /**
7277  * enum kbasep_pm_action - Actions that can be performed on a core.
7378  *
74
- * This enumeration is private to the file. Its values are set to allow
75
- * core_type_to_reg() function, which decodes this enumeration, to be simpler
76
- * and more efficient.
77
- *
7879  * @ACTION_PRESENT: The cores that are present
7980  * @ACTION_READY: The cores that are ready
8081  * @ACTION_PWRON: Power on the cores specified
8182  * @ACTION_PWROFF: Power off the cores specified
8283  * @ACTION_PWRTRANS: The cores that are transitioning
8384  * @ACTION_PWRACTIVE: The cores that are active
85
+ *
86
+ * This enumeration is private to the file. Its values are set to allow
87
+ * core_type_to_reg() function, which decodes this enumeration, to be simpler
88
+ * and more efficient.
8489  */
8590 enum kbasep_pm_action {
8691 	ACTION_PRESENT = 0,
....@@ -96,6 +101,8 @@
96101 		enum kbase_pm_core_type core_type,
97102 		enum kbasep_pm_action action);
98103 
104
+static void kbase_pm_hw_issues_apply(struct kbase_device *kbdev);
105
+
99106 #if MALI_USE_CSF
100107 bool kbase_pm_is_mcu_desired(struct kbase_device *kbdev)
101108 {
....@@ -104,8 +111,14 @@
104111 	if (unlikely(!kbdev->csf.firmware_inited))
105112 		return false;
106113 
107
-	if (kbdev->csf.scheduler.pm_active_count)
114
+	if (kbdev->csf.scheduler.pm_active_count &&
115
+	    kbdev->pm.backend.mcu_desired)
108116 		return true;
117
+
118
+#ifdef KBASE_PM_RUNTIME
119
+	if (kbdev->pm.backend.gpu_wakeup_override)
120
+		return true;
121
+#endif
109122 
110123 	/* MCU is supposed to be ON, only when scheduler.pm_active_count is
111124 	 * non zero. But for always_on policy, the MCU needs to be kept on,
....@@ -120,6 +133,7 @@
120133 
121134 bool kbase_pm_is_l2_desired(struct kbase_device *kbdev)
122135 {
136
+#if !MALI_USE_CSF
123137 	if (kbdev->pm.backend.protected_entry_transition_override)
124138 		return false;
125139 
....@@ -130,15 +144,19 @@
130144 	if (kbdev->pm.backend.protected_transition_override &&
131145 			!kbdev->pm.backend.shaders_desired)
132146 		return false;
133
-
134
-#if MALI_USE_CSF
135
-	if (kbdev->pm.backend.policy_change_clamp_state_to_off)
147
+#else
148
+	if (unlikely(kbdev->pm.backend.policy_change_clamp_state_to_off))
136149 		return false;
150
+
151
+	/* Power up the L2 cache only when MCU is desired */
152
+	if (likely(kbdev->csf.firmware_inited))
153
+		return kbase_pm_is_mcu_desired(kbdev);
137154 #endif
138155 
139156 	return kbdev->pm.backend.l2_desired;
140157 }
141158 
159
+#if !MALI_USE_CSF
142160 void kbase_pm_protected_override_enable(struct kbase_device *kbdev)
143161 {
144162 	lockdep_assert_held(&kbdev->hwaccess_lock);
....@@ -204,17 +222,18 @@
204222 
205223 	kbase_pm_update_state(kbdev);
206224 }
225
+#endif
207226 
208227 /**
209228  * core_type_to_reg - Decode a core type and action to a register.
229
+ *
230
+ * @core_type: The type of core
231
+ * @action:    The type of action
210232  *
211233  * Given a core type (defined by kbase_pm_core_type) and an action (defined
212234  * by kbasep_pm_action) this function will return the register offset that
213235  * will perform the action on the core type. The register returned is the _LO
214236  * register and an offset must be applied to use the _HI register.
215
- *
216
- * @core_type: The type of core
217
- * @action:    The type of action
218237  *
219238  * Return: The register offset of the _LO register that performs an action of
220239  * type @action on a core of type @core_type.
....@@ -259,9 +278,8 @@
259278 	 * to be called from.
260279 	 */
261280 
262
-	kbase_reg_write(kbdev,
263
-			GPU_CONTROL_REG(GPU_COMMAND),
264
-			GPU_COMMAND_CLEAN_INV_CACHES);
281
+	kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
282
+			GPU_COMMAND_CACHE_CLN_INV_L2);
265283 
266284 	raw = kbase_reg_read(kbdev,
267285 		GPU_CONTROL_REG(GPU_IRQ_RAWSTAT));
....@@ -279,14 +297,14 @@
279297 /**
280298  * kbase_pm_invoke - Invokes an action on a core set
281299  *
282
- * This function performs the action given by @action on a set of cores of a
283
- * type given by @core_type. It is a static function used by
284
- * kbase_pm_transition_core_type()
285
- *
286300  * @kbdev:     The kbase device structure of the device
287301  * @core_type: The type of core that the action should be performed on
288302  * @cores:     A bit mask of cores to perform the action on (low 32 bits)
289303  * @action:    The action to perform on the cores
304
+ *
305
+ * This function performs the action given by @action on a set of cores of a
306
+ * type given by @core_type. It is a static function used by
307
+ * kbase_pm_transition_core_type()
290308  */
291309 static void kbase_pm_invoke(struct kbase_device *kbdev,
292310 					enum kbase_pm_core_type core_type,
....@@ -364,14 +382,14 @@
364382 /**
365383  * kbase_pm_get_state - Get information about a core set
366384  *
385
+ * @kbdev:     The kbase device structure of the device
386
+ * @core_type: The type of core that the should be queried
387
+ * @action:    The property of the cores to query
388
+ *
367389  * This function gets information (chosen by @action) about a set of cores of
368390  * a type given by @core_type. It is a static function used by
369391  * kbase_pm_get_active_cores(), kbase_pm_get_trans_cores() and
370392  * kbase_pm_get_ready_cores().
371
- *
372
- * @kbdev:     The kbase device structure of the device
373
- * @core_type: The type of core that the should be queried
374
- * @action:    The property of the cores to query
375393  *
376394  * Return: A bit mask specifying the state of the cores
377395  */
....@@ -520,6 +538,14 @@
520538 	if (!kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_L2_CONFIG))
521539 		return;
522540 
541
+#if MALI_USE_CSF
542
+	if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PBHA_HWU)) {
543
+		val = kbase_reg_read(kbdev, GPU_CONTROL_REG(L2_CONFIG));
544
+		kbase_reg_write(kbdev, GPU_CONTROL_REG(L2_CONFIG),
545
+				L2_CONFIG_PBHA_HWU_SET(val, kbdev->pbha_propagate_bits));
546
+	}
547
+#endif /* MALI_USE_CSF */
548
+
523549 	/*
524550 	 * Skip if size and hash are not given explicitly,
525551 	 * which means default values are used.
....@@ -581,6 +607,21 @@
581607 		return strings[state];
582608 }
583609 
610
+static
611
+void kbase_ktrace_log_mcu_state(struct kbase_device *kbdev, enum kbase_mcu_state state)
612
+{
613
+#if KBASE_KTRACE_ENABLE
614
+	switch (state) {
615
+#define KBASEP_MCU_STATE(n) \
616
+	case KBASE_MCU_ ## n: \
617
+		KBASE_KTRACE_ADD(kbdev, PM_MCU_ ## n, NULL, state); \
618
+		break;
619
+#include "mali_kbase_pm_mcu_states.h"
620
+#undef KBASEP_MCU_STATE
621
+	}
622
+#endif
623
+}
624
+
584625 static inline bool kbase_pm_handle_mcu_core_attr_update(struct kbase_device *kbdev)
585626 {
586627 	struct kbase_pm_backend_data *backend = &kbdev->pm.backend;
....@@ -610,6 +651,97 @@
610651 	return (core_mask_update || timer_update);
611652 }
612653 
654
+bool kbase_pm_is_mcu_inactive(struct kbase_device *kbdev,
655
+			      enum kbase_mcu_state state)
656
+{
657
+	lockdep_assert_held(&kbdev->hwaccess_lock);
658
+
659
+	return ((state == KBASE_MCU_OFF) || (state == KBASE_MCU_IN_SLEEP));
660
+}
661
+
662
+#ifdef KBASE_PM_RUNTIME
663
+/**
664
+ * kbase_pm_enable_mcu_db_notification - Enable the Doorbell notification on
665
+ *                                       MCU side
666
+ *
667
+ * @kbdev: Pointer to the device.
668
+ *
669
+ * This function is called to re-enable the Doorbell notification on MCU side
670
+ * when MCU needs to beome active again.
671
+ */
672
+static void kbase_pm_enable_mcu_db_notification(struct kbase_device *kbdev)
673
+{
674
+	u32 val = kbase_reg_read(kbdev, GPU_CONTROL_REG(MCU_CONTROL));
675
+
676
+	lockdep_assert_held(&kbdev->hwaccess_lock);
677
+
678
+	val &= ~MCU_CNTRL_DOORBELL_DISABLE_MASK;
679
+	kbase_reg_write(kbdev, GPU_CONTROL_REG(MCU_CONTROL), val);
680
+}
681
+
682
+/**
683
+ * wait_mcu_as_inactive - Wait for AS used by MCU FW to get configured
684
+ *
685
+ * @kbdev: Pointer to the device.
686
+ *
687
+ * This function is called to wait for the AS used by MCU FW to get configured
688
+ * before DB notification on MCU is enabled, as a workaround for HW issue.
689
+ */
690
+static void wait_mcu_as_inactive(struct kbase_device *kbdev)
691
+{
692
+	unsigned int max_loops = KBASE_AS_INACTIVE_MAX_LOOPS;
693
+
694
+	lockdep_assert_held(&kbdev->hwaccess_lock);
695
+
696
+	if (!kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TURSEHW_2716))
697
+		return;
698
+
699
+	/* Wait for the AS_ACTIVE_INT bit to become 0 for the AS used by MCU FW */
700
+	while (--max_loops &&
701
+	       kbase_reg_read(kbdev, MMU_AS_REG(MCU_AS_NR, AS_STATUS)) &
702
+			      AS_STATUS_AS_ACTIVE_INT)
703
+		;
704
+
705
+	if (!WARN_ON_ONCE(max_loops == 0))
706
+		return;
707
+
708
+	dev_err(kbdev->dev, "AS_ACTIVE_INT bit stuck for AS %d used by MCU FW", MCU_AS_NR);
709
+
710
+	if (kbase_prepare_to_reset_gpu(kbdev, 0))
711
+		kbase_reset_gpu(kbdev);
712
+}
713
+#endif
714
+
715
+/**
716
+ * kbasep_pm_toggle_power_interrupt - Toggles the IRQ mask for power interrupts
717
+ *                                    from the firmware
718
+ *
719
+ * @kbdev:  Pointer to the device
720
+ * @enable: boolean indicating to enable interrupts or not
721
+ *
722
+ * The POWER_CHANGED_ALL interrupt can be disabled after L2 has been turned on
723
+ * when FW is controlling the power for the shader cores. Correspondingly, the
724
+ * interrupts can be re-enabled after the MCU has been disabled before the
725
+ * power down of L2.
726
+ */
727
+static void kbasep_pm_toggle_power_interrupt(struct kbase_device *kbdev, bool enable)
728
+{
729
+	u32 irq_mask;
730
+
731
+	lockdep_assert_held(&kbdev->hwaccess_lock);
732
+
733
+	irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK));
734
+
735
+	if (enable) {
736
+		irq_mask |= POWER_CHANGED_ALL;
737
+		kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), POWER_CHANGED_ALL);
738
+	} else {
739
+		irq_mask &= ~POWER_CHANGED_ALL;
740
+	}
741
+
742
+	kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), irq_mask);
743
+}
744
+
613745 static int kbase_pm_mcu_update_state(struct kbase_device *kbdev)
614746 {
615747 	struct kbase_pm_backend_data *backend = &kbdev->pm.backend;
....@@ -618,12 +750,12 @@
618750 	lockdep_assert_held(&kbdev->hwaccess_lock);
619751 
620752 	/*
621
-	 * Initial load of firmare should have been done to
753
+	 * Initial load of firmware should have been done to
622754 	 * exercise the MCU state machine.
623755 	 */
624756 	if (unlikely(!kbdev->csf.firmware_inited)) {
625757 		WARN_ON(backend->mcu_state != KBASE_MCU_OFF);
626
-		return -EIO;
758
+		return 0;
627759 	}
628760 
629761 	do {
....@@ -653,6 +785,8 @@
653785 					kbase_pm_ca_get_core_mask(kbdev);
654786 				kbase_csf_firmware_global_reinit(kbdev,
655787 					backend->shaders_desired_mask);
788
+				if (!kbdev->csf.firmware_hctl_core_pwr)
789
+					kbasep_pm_toggle_power_interrupt(kbdev, false);
656790 				backend->mcu_state =
657791 					KBASE_MCU_ON_GLB_REINIT_PEND;
658792 			}
....@@ -670,6 +804,17 @@
670804 						KBASE_MCU_HCTL_SHADERS_PEND_ON;
671805 				} else
672806 					backend->mcu_state = KBASE_MCU_ON_HWCNT_ENABLE;
807
+#if IS_ENABLED(CONFIG_MALI_CORESIGHT)
808
+				if (kbase_debug_coresight_csf_state_check(
809
+					    kbdev, KBASE_DEBUG_CORESIGHT_CSF_DISABLED)) {
810
+					kbase_debug_coresight_csf_state_request(
811
+						kbdev, KBASE_DEBUG_CORESIGHT_CSF_ENABLED);
812
+					backend->mcu_state = KBASE_MCU_CORESIGHT_ENABLE;
813
+				} else if (kbase_debug_coresight_csf_state_check(
814
+						   kbdev, KBASE_DEBUG_CORESIGHT_CSF_ENABLED)) {
815
+					backend->mcu_state = KBASE_MCU_CORESIGHT_ENABLE;
816
+				}
817
+#endif /* IS_ENABLED(CONFIG_MALI_CORESIGHT) */
673818 			}
674819 			break;
675820 
....@@ -698,8 +843,7 @@
698843 				unsigned long flags;
699844 
700845 				kbase_csf_scheduler_spin_lock(kbdev, &flags);
701
-				kbase_hwcnt_context_enable(
702
-					kbdev->hwcnt_gpu_ctx);
846
+				kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx);
703847 				kbase_csf_scheduler_spin_unlock(kbdev, flags);
704848 				backend->hwcnt_disabled = false;
705849 			}
....@@ -712,8 +856,8 @@
712856 			if (!kbase_pm_is_mcu_desired(kbdev))
713857 				backend->mcu_state = KBASE_MCU_ON_HWCNT_DISABLE;
714858 			else if (kbdev->csf.firmware_hctl_core_pwr) {
715
-				/* Host control add additional Cores to be active */
716
-				if (backend->shaders_desired_mask & ~shaders_ready) {
859
+				/* Host control scale up/down cores as needed */
860
+				if (backend->shaders_desired_mask != shaders_ready) {
717861 					backend->hwcnt_desired = false;
718862 					if (!backend->hwcnt_disabled)
719863 						kbase_pm_trigger_hwcnt_disable(kbdev);
....@@ -721,8 +865,18 @@
721865 						KBASE_MCU_HCTL_MCU_ON_RECHECK;
722866 				}
723867 			} else if (kbase_pm_handle_mcu_core_attr_update(kbdev))
724
-				kbdev->pm.backend.mcu_state =
725
-					KBASE_MCU_ON_CORE_ATTR_UPDATE_PEND;
868
+				backend->mcu_state = KBASE_MCU_ON_CORE_ATTR_UPDATE_PEND;
869
+#if IS_ENABLED(CONFIG_MALI_CORESIGHT)
870
+			else if (kbdev->csf.coresight.disable_on_pmode_enter) {
871
+				kbase_debug_coresight_csf_state_request(
872
+					kbdev, KBASE_DEBUG_CORESIGHT_CSF_DISABLED);
873
+				backend->mcu_state = KBASE_MCU_ON_PMODE_ENTER_CORESIGHT_DISABLE;
874
+			} else if (kbdev->csf.coresight.enable_on_pmode_exit) {
875
+				kbase_debug_coresight_csf_state_request(
876
+					kbdev, KBASE_DEBUG_CORESIGHT_CSF_ENABLED);
877
+				backend->mcu_state = KBASE_MCU_ON_PMODE_EXIT_CORESIGHT_ENABLE;
878
+			}
879
+#endif
726880 			break;
727881 
728882 		case KBASE_MCU_HCTL_MCU_ON_RECHECK:
....@@ -746,16 +900,54 @@
746900 						ACTION_PWRON);
747901 				backend->mcu_state =
748902 					KBASE_MCU_HCTL_SHADERS_PEND_ON;
903
+
904
+			} else if (~backend->shaders_desired_mask & shaders_ready) {
905
+				kbase_csf_firmware_update_core_attr(kbdev, false, true,
906
+								    backend->shaders_desired_mask);
907
+				backend->mcu_state = KBASE_MCU_HCTL_CORES_DOWN_SCALE_NOTIFY_PEND;
749908 			} else {
750909 				backend->mcu_state =
751910 					KBASE_MCU_HCTL_SHADERS_PEND_ON;
752911 			}
753912 			break;
754913 
914
+		case KBASE_MCU_HCTL_CORES_DOWN_SCALE_NOTIFY_PEND:
915
+			if (kbase_csf_firmware_core_attr_updated(kbdev)) {
916
+				/* wait in queue until cores idle */
917
+				queue_work(backend->core_idle_wq, &backend->core_idle_work);
918
+				backend->mcu_state = KBASE_MCU_HCTL_CORE_INACTIVE_PEND;
919
+			}
920
+			break;
921
+
922
+		case KBASE_MCU_HCTL_CORE_INACTIVE_PEND:
923
+			{
924
+				u64 active_cores = kbase_pm_get_active_cores(
925
+							kbdev,
926
+							KBASE_PM_CORE_SHADER);
927
+				u64 cores_to_disable = shaders_ready &
928
+							~backend->shaders_desired_mask;
929
+
930
+				if (!(cores_to_disable & active_cores)) {
931
+					kbase_pm_invoke(kbdev, KBASE_PM_CORE_SHADER,
932
+							cores_to_disable,
933
+							ACTION_PWROFF);
934
+					backend->shaders_avail = backend->shaders_desired_mask;
935
+					backend->mcu_state = KBASE_MCU_HCTL_SHADERS_CORE_OFF_PEND;
936
+				}
937
+			}
938
+			break;
939
+
940
+		case KBASE_MCU_HCTL_SHADERS_CORE_OFF_PEND:
941
+			if (!shaders_trans && shaders_ready == backend->shaders_avail) {
942
+				/* Cores now stable */
943
+				backend->pm_shaders_core_mask = shaders_ready;
944
+				backend->mcu_state = KBASE_MCU_ON_HWCNT_ENABLE;
945
+			}
946
+			break;
947
+
755948 		case KBASE_MCU_ON_CORE_ATTR_UPDATE_PEND:
756949 			if (kbase_csf_firmware_core_attr_updated(kbdev)) {
757
-				backend->shaders_avail =
758
-					backend->shaders_desired_mask;
950
+				backend->shaders_avail = backend->shaders_desired_mask;
759951 				backend->mcu_state = KBASE_MCU_ON;
760952 			}
761953 			break;
....@@ -770,9 +962,50 @@
770962 			if (!backend->hwcnt_disabled)
771963 				kbase_pm_trigger_hwcnt_disable(kbdev);
772964 
773
-			if (backend->hwcnt_disabled)
965
+
966
+			if (backend->hwcnt_disabled) {
967
+#ifdef KBASE_PM_RUNTIME
968
+				if (backend->gpu_sleep_mode_active)
969
+					backend->mcu_state = KBASE_MCU_ON_SLEEP_INITIATE;
970
+				else {
971
+#endif
972
+					backend->mcu_state = KBASE_MCU_ON_HALT;
973
+#if IS_ENABLED(CONFIG_MALI_CORESIGHT)
974
+					kbase_debug_coresight_csf_state_request(
975
+						kbdev, KBASE_DEBUG_CORESIGHT_CSF_DISABLED);
976
+					backend->mcu_state = KBASE_MCU_CORESIGHT_DISABLE;
977
+#endif /* IS_ENABLED(CONFIG_MALI_CORESIGHT) */
978
+				}
979
+			}
980
+			break;
981
+
982
+#if IS_ENABLED(CONFIG_MALI_CORESIGHT)
983
+		case KBASE_MCU_ON_PMODE_ENTER_CORESIGHT_DISABLE:
984
+			if (kbase_debug_coresight_csf_state_check(
985
+				    kbdev, KBASE_DEBUG_CORESIGHT_CSF_DISABLED)) {
986
+				backend->mcu_state = KBASE_MCU_ON;
987
+				kbdev->csf.coresight.disable_on_pmode_enter = false;
988
+			}
989
+			break;
990
+		case KBASE_MCU_ON_PMODE_EXIT_CORESIGHT_ENABLE:
991
+			if (kbase_debug_coresight_csf_state_check(
992
+				    kbdev, KBASE_DEBUG_CORESIGHT_CSF_ENABLED)) {
993
+				backend->mcu_state = KBASE_MCU_ON;
994
+				kbdev->csf.coresight.enable_on_pmode_exit = false;
995
+			}
996
+			break;
997
+		case KBASE_MCU_CORESIGHT_DISABLE:
998
+			if (kbase_debug_coresight_csf_state_check(
999
+				    kbdev, KBASE_DEBUG_CORESIGHT_CSF_DISABLED))
7741000 				backend->mcu_state = KBASE_MCU_ON_HALT;
7751001 			break;
1002
+
1003
+		case KBASE_MCU_CORESIGHT_ENABLE:
1004
+			if (kbase_debug_coresight_csf_state_check(
1005
+				    kbdev, KBASE_DEBUG_CORESIGHT_CSF_ENABLED))
1006
+				backend->mcu_state = KBASE_MCU_ON_HWCNT_ENABLE;
1007
+			break;
1008
+#endif /* IS_ENABLED(CONFIG_MALI_CORESIGHT) */
7761009 
7771010 		case KBASE_MCU_ON_HALT:
7781011 			if (!kbase_pm_is_mcu_desired(kbdev)) {
....@@ -784,6 +1017,8 @@
7841017 
7851018 		case KBASE_MCU_ON_PEND_HALT:
7861019 			if (kbase_csf_firmware_mcu_halted(kbdev)) {
1020
+				KBASE_KTRACE_ADD(kbdev, CSF_FIRMWARE_MCU_HALTED, NULL,
1021
+					kbase_csf_ktrace_gpu_cycle_cnt(kbdev));
7871022 				if (kbdev->csf.firmware_hctl_core_pwr)
7881023 					backend->mcu_state =
7891024 						KBASE_MCU_HCTL_SHADERS_READY_OFF;
....@@ -814,13 +1049,61 @@
8141049 		case KBASE_MCU_PEND_OFF:
8151050 			/* wait synchronously for the MCU to get disabled */
8161051 			kbase_csf_firmware_disable_mcu_wait(kbdev);
1052
+			if (!kbdev->csf.firmware_hctl_core_pwr)
1053
+				kbasep_pm_toggle_power_interrupt(kbdev, true);
8171054 			backend->mcu_state = KBASE_MCU_OFF;
8181055 			break;
1056
+#ifdef KBASE_PM_RUNTIME
1057
+		case KBASE_MCU_ON_SLEEP_INITIATE:
1058
+			if (!kbase_pm_is_mcu_desired(kbdev)) {
1059
+				kbase_csf_firmware_trigger_mcu_sleep(kbdev);
1060
+				backend->mcu_state = KBASE_MCU_ON_PEND_SLEEP;
1061
+			} else
1062
+				backend->mcu_state = KBASE_MCU_ON_HWCNT_ENABLE;
1063
+			break;
8191064 
1065
+		case KBASE_MCU_ON_PEND_SLEEP:
1066
+			if (kbase_csf_firmware_is_mcu_in_sleep(kbdev)) {
1067
+				KBASE_KTRACE_ADD(kbdev, CSF_FIRMWARE_MCU_SLEEP, NULL,
1068
+					kbase_csf_ktrace_gpu_cycle_cnt(kbdev));
1069
+				backend->mcu_state = KBASE_MCU_IN_SLEEP;
1070
+				kbase_pm_enable_db_mirror_interrupt(kbdev);
1071
+				kbase_csf_scheduler_reval_idleness_post_sleep(kbdev);
1072
+				/* Enable PM interrupt, after MCU has been put
1073
+				 * to sleep, for the power down of L2.
1074
+				 */
1075
+				if (!kbdev->csf.firmware_hctl_core_pwr)
1076
+					kbasep_pm_toggle_power_interrupt(kbdev, true);
1077
+			}
1078
+			break;
1079
+
1080
+		case KBASE_MCU_IN_SLEEP:
1081
+			if (kbase_pm_is_mcu_desired(kbdev) &&
1082
+			    backend->l2_state == KBASE_L2_ON) {
1083
+				wait_mcu_as_inactive(kbdev);
1084
+				KBASE_TLSTREAM_TL_KBASE_CSFFW_FW_REQUEST_WAKEUP(
1085
+					kbdev, kbase_backend_get_cycle_cnt(kbdev));
1086
+				kbase_pm_enable_mcu_db_notification(kbdev);
1087
+				kbase_pm_disable_db_mirror_interrupt(kbdev);
1088
+				/* Disable PM interrupt after L2 has been
1089
+				 * powered up for the wakeup of MCU.
1090
+				 */
1091
+				if (!kbdev->csf.firmware_hctl_core_pwr)
1092
+					kbasep_pm_toggle_power_interrupt(kbdev, false);
1093
+				backend->mcu_state = KBASE_MCU_ON_HWCNT_ENABLE;
1094
+				kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR);
1095
+			}
1096
+			break;
1097
+#endif
8201098 		case KBASE_MCU_RESET_WAIT:
8211099 			/* Reset complete  */
8221100 			if (!backend->in_reset)
8231101 				backend->mcu_state = KBASE_MCU_OFF;
1102
+
1103
+#if IS_ENABLED(CONFIG_MALI_CORESIGHT)
1104
+			kbdev->csf.coresight.disable_on_pmode_enter = false;
1105
+			kbdev->csf.coresight.enable_on_pmode_exit = false;
1106
+#endif /* IS_ENABLED(CONFIG_MALI_CORESIGHT) */
8241107 			break;
8251108 
8261109 		default:
....@@ -828,14 +1111,43 @@
8281111 			     backend->mcu_state);
8291112 		}
8301113 
831
-		if (backend->mcu_state != prev_state)
1114
+		if (backend->mcu_state != prev_state) {
8321115 			dev_dbg(kbdev->dev, "MCU state transition: %s to %s\n",
8331116 				kbase_mcu_state_to_string(prev_state),
8341117 				kbase_mcu_state_to_string(backend->mcu_state));
1118
+			kbase_ktrace_log_mcu_state(kbdev, backend->mcu_state);
1119
+		}
8351120 
8361121 	} while (backend->mcu_state != prev_state);
8371122 
8381123 	return 0;
1124
+}
1125
+
1126
+static void core_idle_worker(struct work_struct *work)
1127
+{
1128
+	struct kbase_device *kbdev =
1129
+		container_of(work, struct kbase_device, pm.backend.core_idle_work);
1130
+	struct kbase_pm_backend_data *backend = &kbdev->pm.backend;
1131
+	unsigned long flags;
1132
+
1133
+	spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
1134
+	while (backend->gpu_powered && (backend->mcu_state == KBASE_MCU_HCTL_CORE_INACTIVE_PEND)) {
1135
+		const unsigned int core_inactive_wait_ms = 1;
1136
+		u64 active_cores = kbase_pm_get_active_cores(kbdev, KBASE_PM_CORE_SHADER);
1137
+		u64 shaders_ready = kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_SHADER);
1138
+		u64 cores_to_disable = shaders_ready & ~backend->shaders_desired_mask;
1139
+
1140
+		if (!(cores_to_disable & active_cores)) {
1141
+			kbase_pm_update_state(kbdev);
1142
+			break;
1143
+		}
1144
+
1145
+		spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
1146
+		msleep(core_inactive_wait_ms);
1147
+		spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
1148
+	}
1149
+
1150
+	spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
8391151 }
8401152 #endif
8411153 
....@@ -852,13 +1164,77 @@
8521164 		return strings[state];
8531165 }
8541166 
1167
+static
1168
+void kbase_ktrace_log_l2_core_state(struct kbase_device *kbdev, enum kbase_l2_core_state state)
1169
+{
1170
+#if KBASE_KTRACE_ENABLE
1171
+	switch (state) {
1172
+#define KBASEP_L2_STATE(n) \
1173
+	case KBASE_L2_ ## n: \
1174
+		KBASE_KTRACE_ADD(kbdev, PM_L2_ ## n, NULL, state); \
1175
+		break;
1176
+#include "mali_kbase_pm_l2_states.h"
1177
+#undef KBASEP_L2_STATE
1178
+	}
1179
+#endif
1180
+}
1181
+
1182
+#if !MALI_USE_CSF
1183
+/* On powering on the L2, the tracked kctx becomes stale and can be cleared.
1184
+ * This enables the backend to spare the START_FLUSH.INV_SHADER_OTHER
1185
+ * operation on the first submitted katom after the L2 powering on.
1186
+ */
1187
+static void kbase_pm_l2_clear_backend_slot_submit_kctx(struct kbase_device *kbdev)
1188
+{
1189
+	int js;
1190
+
1191
+	lockdep_assert_held(&kbdev->hwaccess_lock);
1192
+
1193
+	/* Clear the slots' last katom submission kctx */
1194
+	for (js = 0; js < kbdev->gpu_props.num_job_slots; js++)
1195
+		kbdev->hwaccess.backend.slot_rb[js].last_kctx_tagged = SLOT_RB_NULL_TAG_VAL;
1196
+}
1197
+#endif
1198
+
1199
+static bool can_power_down_l2(struct kbase_device *kbdev)
1200
+{
1201
+#if MALI_USE_CSF
1202
+	/* Due to the HW issue GPU2019-3878, need to prevent L2 power off
1203
+	 * whilst MMU command is in progress.
1204
+	 * Also defer the power-down if MMU is in process of page migration.
1205
+	 */
1206
+	return !kbdev->mmu_hw_operation_in_progress && !kbdev->mmu_page_migrate_in_progress;
1207
+#else
1208
+	return !kbdev->mmu_page_migrate_in_progress;
1209
+#endif
1210
+}
1211
+
1212
+static bool can_power_up_l2(struct kbase_device *kbdev)
1213
+{
1214
+	lockdep_assert_held(&kbdev->hwaccess_lock);
1215
+
1216
+	/* Avoiding l2 transition if MMU is undergoing page migration */
1217
+	return !kbdev->mmu_page_migrate_in_progress;
1218
+}
1219
+
1220
+static bool need_tiler_control(struct kbase_device *kbdev)
1221
+{
1222
+#if MALI_USE_CSF
1223
+	if (kbase_pm_no_mcu_core_pwroff(kbdev))
1224
+		return true;
1225
+	else
1226
+		return false;
1227
+#else
1228
+	return true;
1229
+#endif
1230
+}
1231
+
8551232 static int kbase_pm_l2_update_state(struct kbase_device *kbdev)
8561233 {
8571234 	struct kbase_pm_backend_data *backend = &kbdev->pm.backend;
8581235 	u64 l2_present = kbdev->gpu_props.curr_config.l2_present;
859
-#if !MALI_USE_CSF
8601236 	u64 tiler_present = kbdev->gpu_props.props.raw_props.tiler_present;
861
-#endif
1237
+	bool l2_power_up_done;
8621238 	enum kbase_l2_core_state prev_state;
8631239 
8641240 	lockdep_assert_held(&kbdev->hwaccess_lock);
....@@ -870,54 +1246,76 @@
8701246 		u64 l2_ready = kbase_pm_get_ready_cores(kbdev,
8711247 				KBASE_PM_CORE_L2);
8721248 
873
-#if !MALI_USE_CSF
874
-		u64 tiler_trans = kbase_pm_get_trans_cores(kbdev,
875
-				KBASE_PM_CORE_TILER);
876
-		u64 tiler_ready = kbase_pm_get_ready_cores(kbdev,
877
-				KBASE_PM_CORE_TILER);
878
-#endif
879
-
1249
+#ifdef CONFIG_MALI_ARBITER_SUPPORT
8801250 		/*
8811251 		 * kbase_pm_get_ready_cores and kbase_pm_get_trans_cores
8821252 		 * are vulnerable to corruption if gpu is lost
8831253 		 */
884
-		if (kbase_is_gpu_removed(kbdev)
885
-#ifdef CONFIG_MALI_ARBITER_SUPPORT
886
-				|| kbase_pm_is_gpu_lost(kbdev)) {
887
-#else
888
-				) {
889
-#endif
1254
+		if (kbase_is_gpu_removed(kbdev) || kbase_pm_is_gpu_lost(kbdev)) {
8901255 			backend->shaders_state =
8911256 				KBASE_SHADERS_OFF_CORESTACK_OFF;
892
-			backend->l2_state = KBASE_L2_OFF;
893
-			dev_dbg(kbdev->dev, "GPU lost has occurred - L2 off\n");
1257
+			backend->hwcnt_desired = false;
1258
+			if (!backend->hwcnt_disabled) {
1259
+				/* Don't progress until hw counters are disabled
1260
+				 * This may involve waiting for a worker to complete.
1261
+				 * The HW counters backend disable code checks for the
1262
+				 * GPU removed case and will error out without touching
1263
+				 * the hardware. This step is needed to keep the HW
1264
+				 * counters in a consistent state after a GPU lost.
1265
+				 */
1266
+				backend->l2_state =
1267
+					KBASE_L2_ON_HWCNT_DISABLE;
1268
+				KBASE_KTRACE_ADD(kbdev, PM_L2_ON_HWCNT_DISABLE, NULL,
1269
+							backend->l2_state);
1270
+				kbase_pm_trigger_hwcnt_disable(kbdev);
1271
+			}
1272
+
1273
+			if (backend->hwcnt_disabled) {
1274
+				backend->l2_state = KBASE_L2_OFF;
1275
+				KBASE_KTRACE_ADD(kbdev, PM_L2_OFF, NULL, backend->l2_state);
1276
+				dev_dbg(kbdev->dev, "GPU lost has occurred - L2 off\n");
1277
+			}
8941278 			break;
8951279 		}
1280
+#endif
8961281 
8971282 		/* mask off ready from trans in case transitions finished
8981283 		 * between the register reads
8991284 		 */
9001285 		l2_trans &= ~l2_ready;
901
-#if !MALI_USE_CSF
902
-		tiler_trans &= ~tiler_ready;
903
-#endif
1286
+
9041287 		prev_state = backend->l2_state;
9051288 
9061289 		switch (backend->l2_state) {
9071290 		case KBASE_L2_OFF:
908
-			if (kbase_pm_is_l2_desired(kbdev)) {
1291
+			if (kbase_pm_is_l2_desired(kbdev) && can_power_up_l2(kbdev)) {
1292
+#if MALI_USE_CSF && defined(KBASE_PM_RUNTIME)
1293
+				/* Enable HW timer of IPA control before
1294
+				 * L2 cache is powered-up.
1295
+				 */
1296
+				kbase_ipa_control_handle_gpu_sleep_exit(kbdev);
1297
+#endif
9091298 				/*
9101299 				 * Set the desired config for L2 before
9111300 				 * powering it on
9121301 				 */
9131302 				kbase_pm_l2_config_override(kbdev);
914
-#if !MALI_USE_CSF
915
-				/* L2 is required, power on.  Powering on the
916
-				 * tiler will also power the first L2 cache.
917
-				 */
918
-				kbase_pm_invoke(kbdev, KBASE_PM_CORE_TILER,
919
-						tiler_present, ACTION_PWRON);
1303
+				kbase_pbha_write_settings(kbdev);
9201304 
1305
+				/* If Host is controlling the power for shader
1306
+				 * cores, then it also needs to control the
1307
+				 * power for Tiler.
1308
+				 * Powering on the tiler will also power the
1309
+				 * L2 cache.
1310
+				 */
1311
+				if (need_tiler_control(kbdev)) {
1312
+					kbase_pm_invoke(kbdev, KBASE_PM_CORE_TILER, tiler_present,
1313
+							ACTION_PWRON);
1314
+				} else {
1315
+					kbase_pm_invoke(kbdev, KBASE_PM_CORE_L2, l2_present,
1316
+							ACTION_PWRON);
1317
+				}
1318
+#if !MALI_USE_CSF
9211319 				/* If we have more than one L2 cache then we
9221320 				 * must power them on explicitly.
9231321 				 */
....@@ -925,30 +1323,36 @@
9251323 					kbase_pm_invoke(kbdev, KBASE_PM_CORE_L2,
9261324 							l2_present & ~1,
9271325 							ACTION_PWRON);
928
-#else
929
-				/* With CSF firmware, Host driver doesn't need to
930
-				 * handle power management with both shader and tiler cores.
931
-				 * The CSF firmware will power up the cores appropriately.
932
-				 * So only power the l2 cache explicitly.
933
-				 */
934
-				kbase_pm_invoke(kbdev, KBASE_PM_CORE_L2,
935
-						l2_present, ACTION_PWRON);
1326
+				/* Clear backend slot submission kctx */
1327
+				kbase_pm_l2_clear_backend_slot_submit_kctx(kbdev);
9361328 #endif
9371329 				backend->l2_state = KBASE_L2_PEND_ON;
9381330 			}
9391331 			break;
9401332 
9411333 		case KBASE_L2_PEND_ON:
942
-#if !MALI_USE_CSF
943
-			if (!l2_trans && l2_ready == l2_present && !tiler_trans
944
-					&& tiler_ready == tiler_present) {
945
-				KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_TILER, NULL,
946
-						tiler_ready);
947
-#else
1334
+			l2_power_up_done = false;
9481335 			if (!l2_trans && l2_ready == l2_present) {
949
-				KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_L2, NULL,
950
-						l2_ready);
951
-#endif
1336
+				if (need_tiler_control(kbdev)) {
1337
+					u64 tiler_trans = kbase_pm_get_trans_cores(
1338
+						kbdev, KBASE_PM_CORE_TILER);
1339
+					u64 tiler_ready = kbase_pm_get_ready_cores(
1340
+						kbdev, KBASE_PM_CORE_TILER);
1341
+					tiler_trans &= ~tiler_ready;
1342
+
1343
+					if (!tiler_trans && tiler_ready == tiler_present) {
1344
+						KBASE_KTRACE_ADD(kbdev,
1345
+								 PM_CORES_CHANGE_AVAILABLE_TILER,
1346
+								 NULL, tiler_ready);
1347
+						l2_power_up_done = true;
1348
+					}
1349
+				} else {
1350
+					KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_L2, NULL,
1351
+							 l2_ready);
1352
+					l2_power_up_done = true;
1353
+				}
1354
+			}
1355
+			if (l2_power_up_done) {
9521356 				/*
9531357 				 * Ensure snoops are enabled after L2 is powered
9541358 				 * up. Note that kbase keeps track of the snoop
....@@ -1027,7 +1431,8 @@
10271431 					break;
10281432 #else
10291433 				/* Do not power off L2 until the MCU has been stopped */
1030
-				if (backend->mcu_state != KBASE_MCU_OFF)
1434
+				if ((backend->mcu_state != KBASE_MCU_OFF) &&
1435
+				    (backend->mcu_state != KBASE_MCU_IN_SLEEP))
10311436 					break;
10321437 #endif
10331438 
....@@ -1073,9 +1478,8 @@
10731478 			}
10741479 
10751480 			backend->hwcnt_desired = false;
1076
-			if (!backend->hwcnt_disabled) {
1481
+			if (!backend->hwcnt_disabled)
10771482 				kbase_pm_trigger_hwcnt_disable(kbdev);
1078
-			}
10791483 #endif
10801484 
10811485 			if (backend->hwcnt_disabled) {
....@@ -1112,27 +1516,31 @@
11121516 			break;
11131517 
11141518 		case KBASE_L2_POWER_DOWN:
1115
-			if (!backend->l2_always_on)
1116
-				/* Powering off the L2 will also power off the
1117
-				 * tiler.
1118
-				 */
1119
-				kbase_pm_invoke(kbdev, KBASE_PM_CORE_L2,
1120
-						l2_present,
1121
-						ACTION_PWROFF);
1122
-			else
1123
-				/* If L2 cache is powered then we must flush it
1124
-				 * before we power off the GPU. Normally this
1125
-				 * would have been handled when the L2 was
1126
-				 * powered off.
1127
-				 */
1128
-				kbase_gpu_start_cache_clean_nolock(
1129
-						kbdev);
1519
+			if (kbase_pm_is_l2_desired(kbdev))
1520
+				backend->l2_state = KBASE_L2_PEND_ON;
1521
+			else if (can_power_down_l2(kbdev)) {
1522
+				if (!backend->l2_always_on)
1523
+					/* Powering off the L2 will also power off the
1524
+					 * tiler.
1525
+					 */
1526
+					kbase_pm_invoke(kbdev, KBASE_PM_CORE_L2,
1527
+							l2_present,
1528
+							ACTION_PWROFF);
1529
+				else
1530
+					/* If L2 cache is powered then we must flush it
1531
+					 * before we power off the GPU. Normally this
1532
+					 * would have been handled when the L2 was
1533
+					 * powered off.
1534
+					 */
1535
+					kbase_gpu_start_cache_clean_nolock(
1536
+						kbdev, GPU_COMMAND_CACHE_CLN_INV_L2);
11301537 #if !MALI_USE_CSF
1131
-			KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_TILER, NULL, 0u);
1538
+				KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_TILER, NULL, 0u);
11321539 #else
1133
-			KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_L2, NULL, 0u);
1540
+				KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_L2, NULL, 0u);
11341541 #endif
1135
-			backend->l2_state = KBASE_L2_PEND_OFF;
1542
+				backend->l2_state = KBASE_L2_PEND_OFF;
1543
+			}
11361544 			break;
11371545 
11381546 		case KBASE_L2_PEND_OFF:
....@@ -1140,12 +1548,26 @@
11401548 				/* We only need to check the L2 here - if the L2
11411549 				 * is off then the tiler is definitely also off.
11421550 				 */
1143
-				if (!l2_trans && !l2_ready)
1551
+				if (!l2_trans && !l2_ready) {
1552
+#if MALI_USE_CSF && defined(KBASE_PM_RUNTIME)
1553
+					/* Allow clock gating within the GPU and prevent it
1554
+					 * from being seen as active during sleep.
1555
+					 */
1556
+					kbase_ipa_control_handle_gpu_sleep_enter(kbdev);
1557
+#endif
11441558 					/* L2 is now powered off */
11451559 					backend->l2_state = KBASE_L2_OFF;
1560
+				}
11461561 			} else {
1147
-				if (!kbdev->cache_clean_in_progress)
1562
+				if (!kbdev->cache_clean_in_progress) {
1563
+#if MALI_USE_CSF && defined(KBASE_PM_RUNTIME)
1564
+					/* Allow clock gating within the GPU and prevent it
1565
+					 * from being seen as active during sleep.
1566
+					 */
1567
+					kbase_ipa_control_handle_gpu_sleep_enter(kbdev);
1568
+#endif
11481569 					backend->l2_state = KBASE_L2_OFF;
1570
+				}
11491571 			}
11501572 			break;
11511573 
....@@ -1160,11 +1582,13 @@
11601582 					backend->l2_state);
11611583 		}
11621584 
1163
-		if (backend->l2_state != prev_state)
1585
+		if (backend->l2_state != prev_state) {
11641586 			dev_dbg(kbdev->dev, "L2 state transition: %s to %s\n",
11651587 				kbase_l2_core_state_to_string(prev_state),
11661588 				kbase_l2_core_state_to_string(
11671589 					backend->l2_state));
1590
+			kbase_ktrace_log_l2_core_state(kbdev, backend->l2_state);
1591
+		}
11681592 
11691593 	} while (backend->l2_state != prev_state);
11701594 
....@@ -1503,10 +1927,12 @@
15031927 			break;
15041928 
15051929 		case KBASE_SHADERS_WAIT_FINISHED_CORESTACK_ON:
1506
-			shader_poweroff_timer_queue_cancel(kbdev);
1930
+			if (!backend->partial_shaderoff)
1931
+				shader_poweroff_timer_queue_cancel(kbdev);
15071932 
15081933 			if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TTRX_921)) {
1509
-				kbase_gpu_start_cache_clean_nolock(kbdev);
1934
+				kbase_gpu_start_cache_clean_nolock(
1935
+					kbdev, GPU_COMMAND_CACHE_CLN_INV_L2);
15101936 				backend->shaders_state =
15111937 					KBASE_SHADERS_L2_FLUSHING_CORESTACK_ON;
15121938 			} else {
....@@ -1608,7 +2034,7 @@
16082034 
16092035 	return 0;
16102036 }
1611
-#endif
2037
+#endif /* !MALI_USE_CSF */
16122038 
16132039 static bool kbase_pm_is_in_desired_state_nolock(struct kbase_device *kbdev)
16142040 {
....@@ -1616,12 +2042,7 @@
16162042 
16172043 	lockdep_assert_held(&kbdev->hwaccess_lock);
16182044 
1619
-	if (kbase_pm_is_l2_desired(kbdev) &&
1620
-			kbdev->pm.backend.l2_state != KBASE_L2_ON)
1621
-		in_desired_state = false;
1622
-	else if (!kbase_pm_is_l2_desired(kbdev) &&
1623
-			kbdev->pm.backend.l2_state != KBASE_L2_OFF)
1624
-		in_desired_state = false;
2045
+	in_desired_state = kbase_pm_l2_is_in_desired_state(kbdev);
16252046 
16262047 #if !MALI_USE_CSF
16272048 	if (kbdev->pm.backend.shaders_desired &&
....@@ -1631,12 +2052,7 @@
16312052 			kbdev->pm.backend.shaders_state != KBASE_SHADERS_OFF_CORESTACK_OFF)
16322053 		in_desired_state = false;
16332054 #else
1634
-	if (kbase_pm_is_mcu_desired(kbdev) &&
1635
-	    kbdev->pm.backend.mcu_state != KBASE_MCU_ON)
1636
-		in_desired_state = false;
1637
-	else if (!kbase_pm_is_mcu_desired(kbdev) &&
1638
-		 kbdev->pm.backend.mcu_state != KBASE_MCU_OFF)
1639
-		in_desired_state = false;
2055
+	in_desired_state &= kbase_pm_mcu_is_in_desired_state(kbdev);
16402056 #endif
16412057 
16422058 	return in_desired_state;
....@@ -1734,8 +2150,8 @@
17342150 	if (kbase_pm_mcu_update_state(kbdev))
17352151 		return;
17362152 
1737
-	if (prev_mcu_state != KBASE_MCU_OFF &&
1738
-	    kbdev->pm.backend.mcu_state == KBASE_MCU_OFF) {
2153
+	if (!kbase_pm_is_mcu_inactive(kbdev, prev_mcu_state) &&
2154
+	    kbase_pm_is_mcu_inactive(kbdev, kbdev->pm.backend.mcu_state)) {
17392155 		if (kbase_pm_l2_update_state(kbdev))
17402156 			return;
17412157 	}
....@@ -1803,11 +2219,24 @@
18032219 	stt->default_ticks = DEFAULT_PM_POWEROFF_TICK_SHADER;
18042220 	stt->configured_ticks = stt->default_ticks;
18052221 
2222
+#if MALI_USE_CSF
2223
+	kbdev->pm.backend.core_idle_wq = alloc_workqueue("coreoff_wq", WQ_HIGHPRI | WQ_UNBOUND, 1);
2224
+	if (!kbdev->pm.backend.core_idle_wq) {
2225
+		destroy_workqueue(stt->wq);
2226
+		return -ENOMEM;
2227
+	}
2228
+
2229
+	INIT_WORK(&kbdev->pm.backend.core_idle_work, core_idle_worker);
2230
+#endif
2231
+
18062232 	return 0;
18072233 }
18082234 
18092235 void kbase_pm_state_machine_term(struct kbase_device *kbdev)
18102236 {
2237
+#if MALI_USE_CSF
2238
+	destroy_workqueue(kbdev->pm.backend.core_idle_wq);
2239
+#endif
18112240 	hrtimer_cancel(&kbdev->pm.backend.shader_tick_timer.timer);
18122241 	destroy_workqueue(kbdev->pm.backend.shader_tick_timer.wq);
18132242 }
....@@ -1820,6 +2249,7 @@
18202249 
18212250 	backend->in_reset = true;
18222251 	backend->l2_state = KBASE_L2_RESET_WAIT;
2252
+	KBASE_KTRACE_ADD(kbdev, PM_L2_RESET_WAIT, NULL, backend->l2_state);
18232253 #if !MALI_USE_CSF
18242254 	backend->shaders_state = KBASE_SHADERS_RESET_WAIT;
18252255 #else
....@@ -1828,6 +2258,10 @@
18282258 	 */
18292259 	if (likely(kbdev->csf.firmware_inited)) {
18302260 		backend->mcu_state = KBASE_MCU_RESET_WAIT;
2261
+		KBASE_KTRACE_ADD(kbdev, PM_MCU_RESET_WAIT, NULL, backend->mcu_state);
2262
+#ifdef KBASE_PM_RUNTIME
2263
+		backend->exit_gpu_sleep_mode = true;
2264
+#endif
18312265 		kbdev->csf.firmware_reload_needed = true;
18322266 	} else {
18332267 		WARN_ON(backend->mcu_state != KBASE_MCU_OFF);
....@@ -1865,16 +2299,21 @@
18652299 	 */
18662300 	kbase_gpu_cache_clean_wait_complete(kbdev);
18672301 	backend->in_reset = false;
2302
+#if MALI_USE_CSF && defined(KBASE_PM_RUNTIME)
2303
+	backend->gpu_wakeup_override = false;
2304
+#endif
18682305 	kbase_pm_update_state(kbdev);
18692306 
18702307 	spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
18712308 }
18722309 
1873
-/* Timeout for kbase_pm_wait_for_desired_state when wait_event_killable has
1874
- * aborted due to a fatal signal. If the time spent waiting has exceeded this
1875
- * threshold then there is most likely a hardware issue.
2310
+#if !MALI_USE_CSF
2311
+/* Timeout in milliseconds for GPU Power Management to reach the desired
2312
+ * Shader and L2 state. If the time spent waiting has exceeded this threshold
2313
+ * then there is most likely a hardware issue.
18762314  */
18772315 #define PM_TIMEOUT_MS (5000) /* 5s */
2316
+#endif
18782317 
18792318 static void kbase_pm_timed_out(struct kbase_device *kbdev)
18802319 {
....@@ -1949,19 +2388,21 @@
19492388 	spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
19502389 
19512390 #if MALI_USE_CSF
1952
-	timeout = kbase_csf_timeout_in_jiffies(PM_TIMEOUT_MS);
2391
+	timeout = kbase_csf_timeout_in_jiffies(kbase_get_timeout_ms(kbdev, CSF_PM_TIMEOUT));
19532392 #else
19542393 	timeout = msecs_to_jiffies(PM_TIMEOUT_MS);
19552394 #endif
19562395 
19572396 	/* Wait for cores */
19582397 #if KERNEL_VERSION(4, 13, 1) <= LINUX_VERSION_CODE
1959
-	remaining = wait_event_killable_timeout(
2398
+	remaining = wait_event_killable_timeout(kbdev->pm.backend.gpu_in_desired_state_wait,
2399
+						kbase_pm_is_in_desired_state_with_l2_powered(kbdev),
2400
+						timeout);
19602401 #else
19612402 	remaining = wait_event_timeout(
1962
-#endif
19632403 		kbdev->pm.backend.gpu_in_desired_state_wait,
19642404 		kbase_pm_is_in_desired_state_with_l2_powered(kbdev), timeout);
2405
+#endif
19652406 
19662407 	if (!remaining) {
19672408 		kbase_pm_timed_out(kbdev);
....@@ -1981,7 +2422,7 @@
19812422 	unsigned long flags;
19822423 	long remaining;
19832424 #if MALI_USE_CSF
1984
-	long timeout = kbase_csf_timeout_in_jiffies(PM_TIMEOUT_MS);
2425
+	long timeout = kbase_csf_timeout_in_jiffies(kbase_get_timeout_ms(kbdev, CSF_PM_TIMEOUT));
19852426 #else
19862427 	long timeout = msecs_to_jiffies(PM_TIMEOUT_MS);
19872428 #endif
....@@ -2015,6 +2456,66 @@
20152456 	return err;
20162457 }
20172458 KBASE_EXPORT_TEST_API(kbase_pm_wait_for_desired_state);
2459
+
2460
+#if MALI_USE_CSF
2461
+/**
2462
+ * core_mask_update_done - Check if downscaling of shader cores is done
2463
+ *
2464
+ * @kbdev: The kbase device structure for the device.
2465
+ *
2466
+ * This function checks if the downscaling of cores is effectively complete.
2467
+ *
2468
+ * Return: true if the downscale is done.
2469
+ */
2470
+static bool core_mask_update_done(struct kbase_device *kbdev)
2471
+{
2472
+	bool update_done = false;
2473
+	unsigned long flags;
2474
+
2475
+	spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
2476
+	/* If MCU is in stable ON state then it implies that the downscale
2477
+	 * request had completed.
2478
+	 * If MCU is not active then it implies all cores are off, so can
2479
+	 * consider the downscale request as complete.
2480
+	 */
2481
+	if ((kbdev->pm.backend.mcu_state == KBASE_MCU_ON) ||
2482
+	    kbase_pm_is_mcu_inactive(kbdev, kbdev->pm.backend.mcu_state))
2483
+		update_done = true;
2484
+	spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
2485
+
2486
+	return update_done;
2487
+}
2488
+
2489
+int kbase_pm_wait_for_cores_down_scale(struct kbase_device *kbdev)
2490
+{
2491
+	long timeout = kbase_csf_timeout_in_jiffies(kbase_get_timeout_ms(kbdev, CSF_PM_TIMEOUT));
2492
+	long remaining;
2493
+	int err = 0;
2494
+
2495
+	/* Wait for core mask update to complete  */
2496
+#if KERNEL_VERSION(4, 13, 1) <= LINUX_VERSION_CODE
2497
+	remaining = wait_event_killable_timeout(
2498
+		kbdev->pm.backend.gpu_in_desired_state_wait,
2499
+		core_mask_update_done(kbdev), timeout);
2500
+#else
2501
+	remaining = wait_event_timeout(
2502
+		kbdev->pm.backend.gpu_in_desired_state_wait,
2503
+		core_mask_update_done(kbdev), timeout);
2504
+#endif
2505
+
2506
+	if (!remaining) {
2507
+		kbase_pm_timed_out(kbdev);
2508
+		err = -ETIMEDOUT;
2509
+	} else if (remaining < 0) {
2510
+		dev_info(
2511
+			kbdev->dev,
2512
+			"Wait for cores down scaling got interrupted");
2513
+		err = (int)remaining;
2514
+	}
2515
+
2516
+	return err;
2517
+}
2518
+#endif
20182519 
20192520 void kbase_pm_enable_interrupts(struct kbase_device *kbdev)
20202521 {
....@@ -2074,21 +2575,36 @@
20742575 KBASE_EXPORT_TEST_API(kbase_pm_disable_interrupts);
20752576 
20762577 #if MALI_USE_CSF
2578
+/**
2579
+ * update_user_reg_page_mapping - Update the mapping for USER Register page
2580
+ *
2581
+ * @kbdev: The kbase device structure for the device.
2582
+ *
2583
+ * This function must be called to unmap the dummy or real page from USER Register page
2584
+ * mapping whenever GPU is powered up or down. The dummy or real page would get
2585
+ * appropriately mapped in when Userspace reads the LATEST_FLUSH value.
2586
+ */
20772587 static void update_user_reg_page_mapping(struct kbase_device *kbdev)
20782588 {
2589
+	struct kbase_context *kctx, *n;
2590
+
20792591 	lockdep_assert_held(&kbdev->pm.lock);
20802592 
2081
-	if (kbdev->csf.mali_file_inode) {
2082
-		/* This would zap the pte corresponding to the mapping of User
2083
-		 * register page for all the Kbase contexts.
2593
+	mutex_lock(&kbdev->csf.reg_lock);
2594
+	list_for_each_entry_safe(kctx, n, &kbdev->csf.user_reg.list, csf.user_reg.link) {
2595
+		/* This would zap the PTE corresponding to the mapping of User
2596
+		 * Register page of the kbase context. The mapping will be reestablished
2597
+		 * when the context (user process) needs to access to the page.
20842598 		 */
2085
-		unmap_mapping_range(kbdev->csf.mali_file_inode->i_mapping,
2086
-				    BASEP_MEM_CSF_USER_REG_PAGE_HANDLE,
2087
-				    PAGE_SIZE, 1);
2599
+		unmap_mapping_range(kbdev->csf.user_reg.filp->f_inode->i_mapping,
2600
+				    kctx->csf.user_reg.file_offset << PAGE_SHIFT, PAGE_SIZE, 1);
2601
+		list_del_init(&kctx->csf.user_reg.link);
2602
+		dev_dbg(kbdev->dev, "Updated USER Reg page mapping of ctx %d_%d", kctx->tgid,
2603
+			kctx->id);
20882604 	}
2605
+	mutex_unlock(&kbdev->csf.reg_lock);
20892606 }
20902607 #endif
2091
-
20922608 
20932609 /*
20942610  * pmu layout:
....@@ -2098,6 +2614,7 @@
20982614  */
20992615 void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume)
21002616 {
2617
+	struct kbase_pm_backend_data *backend = &kbdev->pm.backend;
21012618 	bool reset_required = is_resume;
21022619 	unsigned long flags;
21032620 
....@@ -2115,7 +2632,13 @@
21152632 	}
21162633 #endif
21172634 
2118
-	if (kbdev->pm.backend.gpu_powered) {
2635
+	if (backend->gpu_powered) {
2636
+#if MALI_USE_CSF && defined(KBASE_PM_RUNTIME)
2637
+		if (backend->gpu_idled) {
2638
+			backend->callback_power_runtime_gpu_active(kbdev);
2639
+			backend->gpu_idled = false;
2640
+		}
2641
+#endif
21192642 		/* Already turned on */
21202643 		if (kbdev->poweroff_pending)
21212644 			kbase_pm_enable_interrupts(kbdev);
....@@ -2128,21 +2651,22 @@
21282651 
21292652 	KBASE_KTRACE_ADD(kbdev, PM_GPU_ON, NULL, 0u);
21302653 
2131
-	if (is_resume && kbdev->pm.backend.callback_power_resume) {
2132
-		kbdev->pm.backend.callback_power_resume(kbdev);
2654
+	if (is_resume && backend->callback_power_resume) {
2655
+		backend->callback_power_resume(kbdev);
21332656 		return;
2134
-	} else if (kbdev->pm.backend.callback_power_on) {
2135
-		reset_required = kbdev->pm.backend.callback_power_on(kbdev);
2657
+	} else if (backend->callback_power_on) {
2658
+		reset_required = backend->callback_power_on(kbdev);
21362659 	}
21372660 
21382661 	spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
2139
-	kbdev->pm.backend.gpu_powered = true;
2662
+	backend->gpu_powered = true;
21402663 	spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
21412664 
21422665 #if MALI_USE_CSF
21432666 	/* GPU has been turned on, can switch to actual register page */
21442667 	update_user_reg_page_mapping(kbdev);
21452668 #endif
2669
+
21462670 
21472671 	if (reset_required) {
21482672 		/* GPU state was lost, reset GPU to ensure it is in a
....@@ -2194,8 +2718,8 @@
21942718 
21952719 	/* Turn on the L2 caches */
21962720 	spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
2197
-	kbdev->pm.backend.gpu_ready = true;
2198
-	kbdev->pm.backend.l2_desired = true;
2721
+	backend->gpu_ready = true;
2722
+	backend->l2_desired = true;
21992723 #if MALI_USE_CSF
22002724 	if (reset_required) {
22012725 		/* GPU reset was done after the power on, so send the post
....@@ -2209,6 +2733,16 @@
22092733 #endif
22102734 	kbase_pm_update_state(kbdev);
22112735 	spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
2736
+
2737
+#if MALI_USE_CSF && defined(KBASE_PM_RUNTIME)
2738
+	/* GPU is now powered up. Invoke the GPU active callback as GPU idle
2739
+	 * callback would have been invoked before the power down.
2740
+	 */
2741
+	if (backend->gpu_idled) {
2742
+		backend->callback_power_runtime_gpu_active(kbdev);
2743
+		backend->gpu_idled = false;
2744
+	}
2745
+#endif
22122746 }
22132747 
22142748 KBASE_EXPORT_TEST_API(kbase_pm_clock_on);
....@@ -2252,19 +2786,22 @@
22522786 	kbase_ipa_control_handle_gpu_power_off(kbdev);
22532787 #endif
22542788 
2255
-	kbdev->pm.backend.gpu_ready = false;
2256
-
2257
-	/* The GPU power may be turned off from this point */
2258
-	kbdev->pm.backend.gpu_powered = false;
2259
-
2789
+	if (kbase_is_gpu_removed(kbdev)
22602790 #ifdef CONFIG_MALI_ARBITER_SUPPORT
2261
-	if (kbase_pm_is_gpu_lost(kbdev)) {
2791
+			|| kbase_pm_is_gpu_lost(kbdev)) {
2792
+#else
2793
+			) {
2794
+#endif
22622795 		/* Ensure we unblock any threads that are stuck waiting
22632796 		 * for the GPU
22642797 		 */
22652798 		kbase_gpu_cache_clean_wait_complete(kbdev);
22662799 	}
2267
-#endif
2800
+
2801
+	kbdev->pm.backend.gpu_ready = false;
2802
+
2803
+	/* The GPU power may be turned off from this point */
2804
+	kbdev->pm.backend.gpu_powered = false;
22682805 
22692806 	spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
22702807 
....@@ -2300,9 +2837,9 @@
23002837 /**
23012838  * kbase_pm_wait_for_reset - Wait for a reset to happen
23022839  *
2303
- * Wait for the %RESET_COMPLETED IRQ to occur, then reset the waiting state.
2304
- *
23052840  * @kbdev: Kbase device
2841
+ *
2842
+ * Wait for the %RESET_COMPLETED IRQ to occur, then reset the waiting state.
23062843  */
23072844 static void kbase_pm_wait_for_reset(struct kbase_device *kbdev)
23082845 {
....@@ -2431,8 +2968,8 @@
24312968 {
24322969 	struct device_node *np = kbdev->dev->of_node;
24332970 	const u32 gpu_id = kbdev->gpu_props.props.raw_props.gpu_id;
2434
-	const u32 prod_id = (gpu_id & GPU_ID_VERSION_PRODUCT_ID) >>
2435
-				GPU_ID_VERSION_PRODUCT_ID_SHIFT;
2971
+	const u32 prod_id =
2972
+		(gpu_id & GPU_ID_VERSION_PRODUCT_ID) >> KBASE_GPU_ID_VERSION_PRODUCT_ID_SHIFT;
24362973 	int error = 0;
24372974 
24382975 	kbdev->hw_quirks_gpu = 0;
....@@ -2770,6 +3307,7 @@
27703307 
27713308 /**
27723309  * kbase_pm_request_gpu_cycle_counter_do_request - Request cycle counters
3310
+ * @kbdev:     The kbase device structure of the device
27733311  *
27743312  * Increase the count of cycle counter users and turn the cycle counters on if
27753313  * they were previously off
....@@ -2780,8 +3318,6 @@
27803318  *
27813319  * When this function is called the l2 cache must be on - i.e., the GPU must be
27823320  * on.
2783
- *
2784
- * @kbdev:     The kbase device structure of the device
27853321  */
27863322 static void
27873323 kbase_pm_request_gpu_cycle_counter_do_request(struct kbase_device *kbdev)
....@@ -2799,11 +3335,13 @@
27993335 		/* This might happen after GPU reset.
28003336 		 * Then counter needs to be kicked.
28013337 		 */
3338
+#if !IS_ENABLED(CONFIG_MALI_BIFROST_NO_MALI)
28023339 		if (!(kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_STATUS)) &
28033340 		      GPU_STATUS_CYCLE_COUNT_ACTIVE)) {
28043341 			kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
28053342 					GPU_COMMAND_CYCLE_COUNT_START);
28063343 		}
3344
+#endif
28073345 	}
28083346 
28093347 	spin_unlock_irqrestore(