~hc/RK356X_SDK_RELEASE.git

..	..	@@ -1,7 +1,7 @@
1	1	// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
2	2	/*
3	3	*
4		- * (C) COPYRIGHT 2010-2021 ARM Limited. All rights reserved.
	4	+ * (C) COPYRIGHT 2010-2022 ARM Limited. All rights reserved.
5	5	*
6	6	* This program is free software and is provided to you under the terms of the
7	7	* GNU General Public License version 2 as published by the Free Software
..	..	@@ -39,7 +39,8 @@
39	39
40	40	#include <mali_kbase_reset_gpu.h>
41	41	#include <mali_kbase_ctx_sched.h>
42		-#include <mali_kbase_hwcnt_context.h>
	42	+#include <hwcnt/mali_kbase_hwcnt_context.h>
	43	+#include <mali_kbase_pbha.h>
43	44	#include <backend/gpu/mali_kbase_cache_policy_backend.h>
44	45	#include <device/mali_kbase_device.h>
45	46	#include <backend/gpu/mali_kbase_irq_internal.h>
..	..	@@ -51,6 +52,10 @@
51	52	#endif /* CONFIG_MALI_ARBITER_SUPPORT */
52	53	#if MALI_USE_CSF
53	54	#include <csf/ipa_control/mali_kbase_csf_ipa_control.h>
	55	+#endif
	56	+
	57	+#if MALI_USE_CSF
	58	+#include <linux/delay.h>
54	59	#endif
55	60
56	61	#include <linux/of.h>
..	..	@@ -71,16 +76,16 @@
71	76	/**
72	77	* enum kbasep_pm_action - Actions that can be performed on a core.
73	78	*
74		- * This enumeration is private to the file. Its values are set to allow
75		- * core_type_to_reg() function, which decodes this enumeration, to be simpler
76		- * and more efficient.
77		- *
78	79	* @ACTION_PRESENT: The cores that are present
79	80	* @ACTION_READY: The cores that are ready
80	81	* @ACTION_PWRON: Power on the cores specified
81	82	* @ACTION_PWROFF: Power off the cores specified
82	83	* @ACTION_PWRTRANS: The cores that are transitioning
83	84	* @ACTION_PWRACTIVE: The cores that are active
	85	+ *
	86	+ * This enumeration is private to the file. Its values are set to allow
	87	+ * core_type_to_reg() function, which decodes this enumeration, to be simpler
	88	+ * and more efficient.
84	89	*/
85	90	enum kbasep_pm_action {
86	91	ACTION_PRESENT = 0,
..	..	@@ -96,6 +101,8 @@
96	101	enum kbase_pm_core_type core_type,
97	102	enum kbasep_pm_action action);
98	103
	104	+static void kbase_pm_hw_issues_apply(struct kbase_device *kbdev);
	105	+
99	106	#if MALI_USE_CSF
100	107	bool kbase_pm_is_mcu_desired(struct kbase_device *kbdev)
101	108	{
..	..	@@ -104,8 +111,14 @@
104	111	if (unlikely(!kbdev->csf.firmware_inited))
105	112	return false;
106	113
107		- if (kbdev->csf.scheduler.pm_active_count)
	114	+ if (kbdev->csf.scheduler.pm_active_count &&
	115	+ kbdev->pm.backend.mcu_desired)
108	116	return true;
	117	+
	118	+#ifdef KBASE_PM_RUNTIME
	119	+ if (kbdev->pm.backend.gpu_wakeup_override)
	120	+ return true;
	121	+#endif
109	122
110	123	/* MCU is supposed to be ON, only when scheduler.pm_active_count is
111	124	* non zero. But for always_on policy, the MCU needs to be kept on,
..	..	@@ -120,6 +133,7 @@
120	133
121	134	bool kbase_pm_is_l2_desired(struct kbase_device *kbdev)
122	135	{
	136	+#if !MALI_USE_CSF
123	137	if (kbdev->pm.backend.protected_entry_transition_override)
124	138	return false;
125	139
..	..	@@ -130,15 +144,19 @@
130	144	if (kbdev->pm.backend.protected_transition_override &&
131	145	!kbdev->pm.backend.shaders_desired)
132	146	return false;
133		-
134		-#if MALI_USE_CSF
135		- if (kbdev->pm.backend.policy_change_clamp_state_to_off)
	147	+#else
	148	+ if (unlikely(kbdev->pm.backend.policy_change_clamp_state_to_off))
136	149	return false;
	150	+
	151	+ /* Power up the L2 cache only when MCU is desired */
	152	+ if (likely(kbdev->csf.firmware_inited))
	153	+ return kbase_pm_is_mcu_desired(kbdev);
137	154	#endif
138	155
139	156	return kbdev->pm.backend.l2_desired;
140	157	}
141	158
	159	+#if !MALI_USE_CSF
142	160	void kbase_pm_protected_override_enable(struct kbase_device *kbdev)
143	161	{
144	162	lockdep_assert_held(&kbdev->hwaccess_lock);
..	..	@@ -204,17 +222,18 @@
204	222
205	223	kbase_pm_update_state(kbdev);
206	224	}
	225	+#endif
207	226
208	227	/**
209	228	* core_type_to_reg - Decode a core type and action to a register.
	229	+ *
	230	+ * @core_type: The type of core
	231	+ * @action: The type of action
210	232	*
211	233	* Given a core type (defined by kbase_pm_core_type) and an action (defined
212	234	* by kbasep_pm_action) this function will return the register offset that
213	235	* will perform the action on the core type. The register returned is the _LO
214	236	* register and an offset must be applied to use the _HI register.
215		- *
216		- * @core_type: The type of core
217		- * @action: The type of action
218	237	*
219	238	* Return: The register offset of the _LO register that performs an action of
220	239	* type @action on a core of type @core_type.
..	..	@@ -259,9 +278,8 @@
259	278	* to be called from.
260	279	*/
261	280
262		- kbase_reg_write(kbdev,
263		- GPU_CONTROL_REG(GPU_COMMAND),
264		- GPU_COMMAND_CLEAN_INV_CACHES);
	281	+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
	282	+ GPU_COMMAND_CACHE_CLN_INV_L2);
265	283
266	284	raw = kbase_reg_read(kbdev,
267	285	GPU_CONTROL_REG(GPU_IRQ_RAWSTAT));
..	..	@@ -279,14 +297,14 @@
279	297	/**
280	298	* kbase_pm_invoke - Invokes an action on a core set
281	299	*
282		- * This function performs the action given by @action on a set of cores of a
283		- * type given by @core_type. It is a static function used by
284		- * kbase_pm_transition_core_type()
285		- *
286	300	* @kbdev: The kbase device structure of the device
287	301	* @core_type: The type of core that the action should be performed on
288	302	* @cores: A bit mask of cores to perform the action on (low 32 bits)
289	303	* @action: The action to perform on the cores
	304	+ *
	305	+ * This function performs the action given by @action on a set of cores of a
	306	+ * type given by @core_type. It is a static function used by
	307	+ * kbase_pm_transition_core_type()
290	308	*/
291	309	static void kbase_pm_invoke(struct kbase_device *kbdev,
292	310	enum kbase_pm_core_type core_type,
..	..	@@ -364,14 +382,14 @@
364	382	/**
365	383	* kbase_pm_get_state - Get information about a core set
366	384	*
	385	+ * @kbdev: The kbase device structure of the device
	386	+ * @core_type: The type of core that the should be queried
	387	+ * @action: The property of the cores to query
	388	+ *
367	389	* This function gets information (chosen by @action) about a set of cores of
368	390	* a type given by @core_type. It is a static function used by
369	391	* kbase_pm_get_active_cores(), kbase_pm_get_trans_cores() and
370	392	* kbase_pm_get_ready_cores().
371		- *
372		- * @kbdev: The kbase device structure of the device
373		- * @core_type: The type of core that the should be queried
374		- * @action: The property of the cores to query
375	393	*
376	394	* Return: A bit mask specifying the state of the cores
377	395	*/
..	..	@@ -520,6 +538,14 @@
520	538	if (!kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_L2_CONFIG))
521	539	return;
522	540
	541	+#if MALI_USE_CSF
	542	+ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PBHA_HWU)) {
	543	+ val = kbase_reg_read(kbdev, GPU_CONTROL_REG(L2_CONFIG));
	544	+ kbase_reg_write(kbdev, GPU_CONTROL_REG(L2_CONFIG),
	545	+ L2_CONFIG_PBHA_HWU_SET(val, kbdev->pbha_propagate_bits));
	546	+ }
	547	+#endif /* MALI_USE_CSF */
	548	+
523	549	/*
524	550	* Skip if size and hash are not given explicitly,
525	551	* which means default values are used.
..	..	@@ -581,6 +607,21 @@
581	607	return strings[state];
582	608	}
583	609
	610	+static
	611	+void kbase_ktrace_log_mcu_state(struct kbase_device *kbdev, enum kbase_mcu_state state)
	612	+{
	613	+#if KBASE_KTRACE_ENABLE
	614	+ switch (state) {
	615	+#define KBASEP_MCU_STATE(n) \
	616	+ case KBASE_MCU_ ## n: \
	617	+ KBASE_KTRACE_ADD(kbdev, PM_MCU_ ## n, NULL, state); \
	618	+ break;
	619	+#include "mali_kbase_pm_mcu_states.h"
	620	+#undef KBASEP_MCU_STATE
	621	+ }
	622	+#endif
	623	+}
	624	+
584	625	static inline bool kbase_pm_handle_mcu_core_attr_update(struct kbase_device *kbdev)
585	626	{
586	627	struct kbase_pm_backend_data *backend = &kbdev->pm.backend;
..	..	@@ -610,6 +651,97 @@
610	651	return (core_mask_update \|\| timer_update);
611	652	}
612	653
	654	+bool kbase_pm_is_mcu_inactive(struct kbase_device *kbdev,
	655	+ enum kbase_mcu_state state)
	656	+{
	657	+ lockdep_assert_held(&kbdev->hwaccess_lock);
	658	+
	659	+ return ((state == KBASE_MCU_OFF) \|\| (state == KBASE_MCU_IN_SLEEP));
	660	+}
	661	+
	662	+#ifdef KBASE_PM_RUNTIME
	663	+/**
	664	+ * kbase_pm_enable_mcu_db_notification - Enable the Doorbell notification on
	665	+ * MCU side
	666	+ *
	667	+ * @kbdev: Pointer to the device.
	668	+ *
	669	+ * This function is called to re-enable the Doorbell notification on MCU side
	670	+ * when MCU needs to beome active again.
	671	+ */
	672	+static void kbase_pm_enable_mcu_db_notification(struct kbase_device *kbdev)
	673	+{
	674	+ u32 val = kbase_reg_read(kbdev, GPU_CONTROL_REG(MCU_CONTROL));
	675	+
	676	+ lockdep_assert_held(&kbdev->hwaccess_lock);
	677	+
	678	+ val &= ~MCU_CNTRL_DOORBELL_DISABLE_MASK;
	679	+ kbase_reg_write(kbdev, GPU_CONTROL_REG(MCU_CONTROL), val);
	680	+}
	681	+
	682	+/**
	683	+ * wait_mcu_as_inactive - Wait for AS used by MCU FW to get configured
	684	+ *
	685	+ * @kbdev: Pointer to the device.
	686	+ *
	687	+ * This function is called to wait for the AS used by MCU FW to get configured
	688	+ * before DB notification on MCU is enabled, as a workaround for HW issue.
	689	+ */
	690	+static void wait_mcu_as_inactive(struct kbase_device *kbdev)
	691	+{
	692	+ unsigned int max_loops = KBASE_AS_INACTIVE_MAX_LOOPS;
	693	+
	694	+ lockdep_assert_held(&kbdev->hwaccess_lock);
	695	+
	696	+ if (!kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TURSEHW_2716))
	697	+ return;
	698	+
	699	+ /* Wait for the AS_ACTIVE_INT bit to become 0 for the AS used by MCU FW */
	700	+ while (--max_loops &&
	701	+ kbase_reg_read(kbdev, MMU_AS_REG(MCU_AS_NR, AS_STATUS)) &
	702	+ AS_STATUS_AS_ACTIVE_INT)
	703	+ ;
	704	+
	705	+ if (!WARN_ON_ONCE(max_loops == 0))
	706	+ return;
	707	+
	708	+ dev_err(kbdev->dev, "AS_ACTIVE_INT bit stuck for AS %d used by MCU FW", MCU_AS_NR);
	709	+
	710	+ if (kbase_prepare_to_reset_gpu(kbdev, 0))
	711	+ kbase_reset_gpu(kbdev);
	712	+}
	713	+#endif
	714	+
	715	+/**
	716	+ * kbasep_pm_toggle_power_interrupt - Toggles the IRQ mask for power interrupts
	717	+ * from the firmware
	718	+ *
	719	+ * @kbdev: Pointer to the device
	720	+ * @enable: boolean indicating to enable interrupts or not
	721	+ *
	722	+ * The POWER_CHANGED_ALL interrupt can be disabled after L2 has been turned on
	723	+ * when FW is controlling the power for the shader cores. Correspondingly, the
	724	+ * interrupts can be re-enabled after the MCU has been disabled before the
	725	+ * power down of L2.
	726	+ */
	727	+static void kbasep_pm_toggle_power_interrupt(struct kbase_device *kbdev, bool enable)
	728	+{
	729	+ u32 irq_mask;
	730	+
	731	+ lockdep_assert_held(&kbdev->hwaccess_lock);
	732	+
	733	+ irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK));
	734	+
	735	+ if (enable) {
	736	+ irq_mask \|= POWER_CHANGED_ALL;
	737	+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), POWER_CHANGED_ALL);
	738	+ } else {
	739	+ irq_mask &= ~POWER_CHANGED_ALL;
	740	+ }
	741	+
	742	+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), irq_mask);
	743	+}
	744	+
613	745	static int kbase_pm_mcu_update_state(struct kbase_device *kbdev)
614	746	{
615	747	struct kbase_pm_backend_data *backend = &kbdev->pm.backend;
..	..	@@ -618,12 +750,12 @@
618	750	lockdep_assert_held(&kbdev->hwaccess_lock);
619	751
620	752	/*
621		- * Initial load of firmare should have been done to
	753	+ * Initial load of firmware should have been done to
622	754	* exercise the MCU state machine.
623	755	*/
624	756	if (unlikely(!kbdev->csf.firmware_inited)) {
625	757	WARN_ON(backend->mcu_state != KBASE_MCU_OFF);
626		- return -EIO;
	758	+ return 0;
627	759	}
628	760
629	761	do {
..	..	@@ -653,6 +785,8 @@
653	785	kbase_pm_ca_get_core_mask(kbdev);
654	786	kbase_csf_firmware_global_reinit(kbdev,
655	787	backend->shaders_desired_mask);
	788	+ if (!kbdev->csf.firmware_hctl_core_pwr)
	789	+ kbasep_pm_toggle_power_interrupt(kbdev, false);
656	790	backend->mcu_state =
657	791	KBASE_MCU_ON_GLB_REINIT_PEND;
658	792	}
..	..	@@ -670,6 +804,17 @@
670	804	KBASE_MCU_HCTL_SHADERS_PEND_ON;
671	805	} else
672	806	backend->mcu_state = KBASE_MCU_ON_HWCNT_ENABLE;
	807	+#if IS_ENABLED(CONFIG_MALI_CORESIGHT)
	808	+ if (kbase_debug_coresight_csf_state_check(
	809	+ kbdev, KBASE_DEBUG_CORESIGHT_CSF_DISABLED)) {
	810	+ kbase_debug_coresight_csf_state_request(
	811	+ kbdev, KBASE_DEBUG_CORESIGHT_CSF_ENABLED);
	812	+ backend->mcu_state = KBASE_MCU_CORESIGHT_ENABLE;
	813	+ } else if (kbase_debug_coresight_csf_state_check(
	814	+ kbdev, KBASE_DEBUG_CORESIGHT_CSF_ENABLED)) {
	815	+ backend->mcu_state = KBASE_MCU_CORESIGHT_ENABLE;
	816	+ }
	817	+#endif /* IS_ENABLED(CONFIG_MALI_CORESIGHT) */
673	818	}
674	819	break;
675	820
..	..	@@ -698,8 +843,7 @@
698	843	unsigned long flags;
699	844
700	845	kbase_csf_scheduler_spin_lock(kbdev, &flags);
701		- kbase_hwcnt_context_enable(
702		- kbdev->hwcnt_gpu_ctx);
	846	+ kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx);
703	847	kbase_csf_scheduler_spin_unlock(kbdev, flags);
704	848	backend->hwcnt_disabled = false;
705	849	}
..	..	@@ -712,8 +856,8 @@
712	856	if (!kbase_pm_is_mcu_desired(kbdev))
713	857	backend->mcu_state = KBASE_MCU_ON_HWCNT_DISABLE;
714	858	else if (kbdev->csf.firmware_hctl_core_pwr) {
715		- /* Host control add additional Cores to be active */
716		- if (backend->shaders_desired_mask & ~shaders_ready) {
	859	+ /* Host control scale up/down cores as needed */
	860	+ if (backend->shaders_desired_mask != shaders_ready) {
717	861	backend->hwcnt_desired = false;
718	862	if (!backend->hwcnt_disabled)
719	863	kbase_pm_trigger_hwcnt_disable(kbdev);
..	..	@@ -721,8 +865,18 @@
721	865	KBASE_MCU_HCTL_MCU_ON_RECHECK;
722	866	}
723	867	} else if (kbase_pm_handle_mcu_core_attr_update(kbdev))
724		- kbdev->pm.backend.mcu_state =
725		- KBASE_MCU_ON_CORE_ATTR_UPDATE_PEND;
	868	+ backend->mcu_state = KBASE_MCU_ON_CORE_ATTR_UPDATE_PEND;
	869	+#if IS_ENABLED(CONFIG_MALI_CORESIGHT)
	870	+ else if (kbdev->csf.coresight.disable_on_pmode_enter) {
	871	+ kbase_debug_coresight_csf_state_request(
	872	+ kbdev, KBASE_DEBUG_CORESIGHT_CSF_DISABLED);
	873	+ backend->mcu_state = KBASE_MCU_ON_PMODE_ENTER_CORESIGHT_DISABLE;
	874	+ } else if (kbdev->csf.coresight.enable_on_pmode_exit) {
	875	+ kbase_debug_coresight_csf_state_request(
	876	+ kbdev, KBASE_DEBUG_CORESIGHT_CSF_ENABLED);
	877	+ backend->mcu_state = KBASE_MCU_ON_PMODE_EXIT_CORESIGHT_ENABLE;
	878	+ }
	879	+#endif
726	880	break;
727	881
728	882	case KBASE_MCU_HCTL_MCU_ON_RECHECK:
..	..	@@ -746,16 +900,54 @@
746	900	ACTION_PWRON);
747	901	backend->mcu_state =
748	902	KBASE_MCU_HCTL_SHADERS_PEND_ON;
	903	+
	904	+ } else if (~backend->shaders_desired_mask & shaders_ready) {
	905	+ kbase_csf_firmware_update_core_attr(kbdev, false, true,
	906	+ backend->shaders_desired_mask);
	907	+ backend->mcu_state = KBASE_MCU_HCTL_CORES_DOWN_SCALE_NOTIFY_PEND;
749	908	} else {
750	909	backend->mcu_state =
751	910	KBASE_MCU_HCTL_SHADERS_PEND_ON;
752	911	}
753	912	break;
754	913
	914	+ case KBASE_MCU_HCTL_CORES_DOWN_SCALE_NOTIFY_PEND:
	915	+ if (kbase_csf_firmware_core_attr_updated(kbdev)) {
	916	+ /* wait in queue until cores idle */
	917	+ queue_work(backend->core_idle_wq, &backend->core_idle_work);
	918	+ backend->mcu_state = KBASE_MCU_HCTL_CORE_INACTIVE_PEND;
	919	+ }
	920	+ break;
	921	+
	922	+ case KBASE_MCU_HCTL_CORE_INACTIVE_PEND:
	923	+ {
	924	+ u64 active_cores = kbase_pm_get_active_cores(
	925	+ kbdev,
	926	+ KBASE_PM_CORE_SHADER);
	927	+ u64 cores_to_disable = shaders_ready &
	928	+ ~backend->shaders_desired_mask;
	929	+
	930	+ if (!(cores_to_disable & active_cores)) {
	931	+ kbase_pm_invoke(kbdev, KBASE_PM_CORE_SHADER,
	932	+ cores_to_disable,
	933	+ ACTION_PWROFF);
	934	+ backend->shaders_avail = backend->shaders_desired_mask;
	935	+ backend->mcu_state = KBASE_MCU_HCTL_SHADERS_CORE_OFF_PEND;
	936	+ }
	937	+ }
	938	+ break;
	939	+
	940	+ case KBASE_MCU_HCTL_SHADERS_CORE_OFF_PEND:
	941	+ if (!shaders_trans && shaders_ready == backend->shaders_avail) {
	942	+ /* Cores now stable */
	943	+ backend->pm_shaders_core_mask = shaders_ready;
	944	+ backend->mcu_state = KBASE_MCU_ON_HWCNT_ENABLE;
	945	+ }
	946	+ break;
	947	+
755	948	case KBASE_MCU_ON_CORE_ATTR_UPDATE_PEND:
756	949	if (kbase_csf_firmware_core_attr_updated(kbdev)) {
757		- backend->shaders_avail =
758		- backend->shaders_desired_mask;
	950	+ backend->shaders_avail = backend->shaders_desired_mask;
759	951	backend->mcu_state = KBASE_MCU_ON;
760	952	}
761	953	break;
..	..	@@ -770,9 +962,50 @@
770	962	if (!backend->hwcnt_disabled)
771	963	kbase_pm_trigger_hwcnt_disable(kbdev);
772	964
773		- if (backend->hwcnt_disabled)
	965	+
	966	+ if (backend->hwcnt_disabled) {
	967	+#ifdef KBASE_PM_RUNTIME
	968	+ if (backend->gpu_sleep_mode_active)
	969	+ backend->mcu_state = KBASE_MCU_ON_SLEEP_INITIATE;
	970	+ else {
	971	+#endif
	972	+ backend->mcu_state = KBASE_MCU_ON_HALT;
	973	+#if IS_ENABLED(CONFIG_MALI_CORESIGHT)
	974	+ kbase_debug_coresight_csf_state_request(
	975	+ kbdev, KBASE_DEBUG_CORESIGHT_CSF_DISABLED);
	976	+ backend->mcu_state = KBASE_MCU_CORESIGHT_DISABLE;
	977	+#endif /* IS_ENABLED(CONFIG_MALI_CORESIGHT) */
	978	+ }
	979	+ }
	980	+ break;
	981	+
	982	+#if IS_ENABLED(CONFIG_MALI_CORESIGHT)
	983	+ case KBASE_MCU_ON_PMODE_ENTER_CORESIGHT_DISABLE:
	984	+ if (kbase_debug_coresight_csf_state_check(
	985	+ kbdev, KBASE_DEBUG_CORESIGHT_CSF_DISABLED)) {
	986	+ backend->mcu_state = KBASE_MCU_ON;
	987	+ kbdev->csf.coresight.disable_on_pmode_enter = false;
	988	+ }
	989	+ break;
	990	+ case KBASE_MCU_ON_PMODE_EXIT_CORESIGHT_ENABLE:
	991	+ if (kbase_debug_coresight_csf_state_check(
	992	+ kbdev, KBASE_DEBUG_CORESIGHT_CSF_ENABLED)) {
	993	+ backend->mcu_state = KBASE_MCU_ON;
	994	+ kbdev->csf.coresight.enable_on_pmode_exit = false;
	995	+ }
	996	+ break;
	997	+ case KBASE_MCU_CORESIGHT_DISABLE:
	998	+ if (kbase_debug_coresight_csf_state_check(
	999	+ kbdev, KBASE_DEBUG_CORESIGHT_CSF_DISABLED))
774	1000	backend->mcu_state = KBASE_MCU_ON_HALT;
775	1001	break;
	1002	+
	1003	+ case KBASE_MCU_CORESIGHT_ENABLE:
	1004	+ if (kbase_debug_coresight_csf_state_check(
	1005	+ kbdev, KBASE_DEBUG_CORESIGHT_CSF_ENABLED))
	1006	+ backend->mcu_state = KBASE_MCU_ON_HWCNT_ENABLE;
	1007	+ break;
	1008	+#endif /* IS_ENABLED(CONFIG_MALI_CORESIGHT) */
776	1009
777	1010	case KBASE_MCU_ON_HALT:
778	1011	if (!kbase_pm_is_mcu_desired(kbdev)) {
..	..	@@ -784,6 +1017,8 @@
784	1017
785	1018	case KBASE_MCU_ON_PEND_HALT:
786	1019	if (kbase_csf_firmware_mcu_halted(kbdev)) {
	1020	+ KBASE_KTRACE_ADD(kbdev, CSF_FIRMWARE_MCU_HALTED, NULL,
	1021	+ kbase_csf_ktrace_gpu_cycle_cnt(kbdev));
787	1022	if (kbdev->csf.firmware_hctl_core_pwr)
788	1023	backend->mcu_state =
789	1024	KBASE_MCU_HCTL_SHADERS_READY_OFF;
..	..	@@ -814,13 +1049,61 @@
814	1049	case KBASE_MCU_PEND_OFF:
815	1050	/* wait synchronously for the MCU to get disabled */
816	1051	kbase_csf_firmware_disable_mcu_wait(kbdev);
	1052	+ if (!kbdev->csf.firmware_hctl_core_pwr)
	1053	+ kbasep_pm_toggle_power_interrupt(kbdev, true);
817	1054	backend->mcu_state = KBASE_MCU_OFF;
818	1055	break;
	1056	+#ifdef KBASE_PM_RUNTIME
	1057	+ case KBASE_MCU_ON_SLEEP_INITIATE:
	1058	+ if (!kbase_pm_is_mcu_desired(kbdev)) {
	1059	+ kbase_csf_firmware_trigger_mcu_sleep(kbdev);
	1060	+ backend->mcu_state = KBASE_MCU_ON_PEND_SLEEP;
	1061	+ } else
	1062	+ backend->mcu_state = KBASE_MCU_ON_HWCNT_ENABLE;
	1063	+ break;
819	1064
	1065	+ case KBASE_MCU_ON_PEND_SLEEP:
	1066	+ if (kbase_csf_firmware_is_mcu_in_sleep(kbdev)) {
	1067	+ KBASE_KTRACE_ADD(kbdev, CSF_FIRMWARE_MCU_SLEEP, NULL,
	1068	+ kbase_csf_ktrace_gpu_cycle_cnt(kbdev));
	1069	+ backend->mcu_state = KBASE_MCU_IN_SLEEP;
	1070	+ kbase_pm_enable_db_mirror_interrupt(kbdev);
	1071	+ kbase_csf_scheduler_reval_idleness_post_sleep(kbdev);
	1072	+ /* Enable PM interrupt, after MCU has been put
	1073	+ * to sleep, for the power down of L2.
	1074	+ */
	1075	+ if (!kbdev->csf.firmware_hctl_core_pwr)
	1076	+ kbasep_pm_toggle_power_interrupt(kbdev, true);
	1077	+ }
	1078	+ break;
	1079	+
	1080	+ case KBASE_MCU_IN_SLEEP:
	1081	+ if (kbase_pm_is_mcu_desired(kbdev) &&
	1082	+ backend->l2_state == KBASE_L2_ON) {
	1083	+ wait_mcu_as_inactive(kbdev);
	1084	+ KBASE_TLSTREAM_TL_KBASE_CSFFW_FW_REQUEST_WAKEUP(
	1085	+ kbdev, kbase_backend_get_cycle_cnt(kbdev));
	1086	+ kbase_pm_enable_mcu_db_notification(kbdev);
	1087	+ kbase_pm_disable_db_mirror_interrupt(kbdev);
	1088	+ /* Disable PM interrupt after L2 has been
	1089	+ * powered up for the wakeup of MCU.
	1090	+ */
	1091	+ if (!kbdev->csf.firmware_hctl_core_pwr)
	1092	+ kbasep_pm_toggle_power_interrupt(kbdev, false);
	1093	+ backend->mcu_state = KBASE_MCU_ON_HWCNT_ENABLE;
	1094	+ kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR);
	1095	+ }
	1096	+ break;
	1097	+#endif
820	1098	case KBASE_MCU_RESET_WAIT:
821	1099	/* Reset complete */
822	1100	if (!backend->in_reset)
823	1101	backend->mcu_state = KBASE_MCU_OFF;
	1102	+
	1103	+#if IS_ENABLED(CONFIG_MALI_CORESIGHT)
	1104	+ kbdev->csf.coresight.disable_on_pmode_enter = false;
	1105	+ kbdev->csf.coresight.enable_on_pmode_exit = false;
	1106	+#endif /* IS_ENABLED(CONFIG_MALI_CORESIGHT) */
824	1107	break;
825	1108
826	1109	default:
..	..	@@ -828,14 +1111,43 @@
828	1111	backend->mcu_state);
829	1112	}
830	1113
831		- if (backend->mcu_state != prev_state)
	1114	+ if (backend->mcu_state != prev_state) {
832	1115	dev_dbg(kbdev->dev, "MCU state transition: %s to %s\n",
833	1116	kbase_mcu_state_to_string(prev_state),
834	1117	kbase_mcu_state_to_string(backend->mcu_state));
	1118	+ kbase_ktrace_log_mcu_state(kbdev, backend->mcu_state);
	1119	+ }
835	1120
836	1121	} while (backend->mcu_state != prev_state);
837	1122
838	1123	return 0;
	1124	+}
	1125	+
	1126	+static void core_idle_worker(struct work_struct *work)
	1127	+{
	1128	+ struct kbase_device *kbdev =
	1129	+ container_of(work, struct kbase_device, pm.backend.core_idle_work);
	1130	+ struct kbase_pm_backend_data *backend = &kbdev->pm.backend;
	1131	+ unsigned long flags;
	1132	+
	1133	+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
	1134	+ while (backend->gpu_powered && (backend->mcu_state == KBASE_MCU_HCTL_CORE_INACTIVE_PEND)) {
	1135	+ const unsigned int core_inactive_wait_ms = 1;
	1136	+ u64 active_cores = kbase_pm_get_active_cores(kbdev, KBASE_PM_CORE_SHADER);
	1137	+ u64 shaders_ready = kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_SHADER);
	1138	+ u64 cores_to_disable = shaders_ready & ~backend->shaders_desired_mask;
	1139	+
	1140	+ if (!(cores_to_disable & active_cores)) {
	1141	+ kbase_pm_update_state(kbdev);
	1142	+ break;
	1143	+ }
	1144	+
	1145	+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
	1146	+ msleep(core_inactive_wait_ms);
	1147	+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
	1148	+ }
	1149	+
	1150	+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
839	1151	}
840	1152	#endif
841	1153
..	..	@@ -852,13 +1164,77 @@
852	1164	return strings[state];
853	1165	}
854	1166
	1167	+static
	1168	+void kbase_ktrace_log_l2_core_state(struct kbase_device *kbdev, enum kbase_l2_core_state state)
	1169	+{
	1170	+#if KBASE_KTRACE_ENABLE
	1171	+ switch (state) {
	1172	+#define KBASEP_L2_STATE(n) \
	1173	+ case KBASE_L2_ ## n: \
	1174	+ KBASE_KTRACE_ADD(kbdev, PM_L2_ ## n, NULL, state); \
	1175	+ break;
	1176	+#include "mali_kbase_pm_l2_states.h"
	1177	+#undef KBASEP_L2_STATE
	1178	+ }
	1179	+#endif
	1180	+}
	1181	+
	1182	+#if !MALI_USE_CSF
	1183	+/* On powering on the L2, the tracked kctx becomes stale and can be cleared.
	1184	+ * This enables the backend to spare the START_FLUSH.INV_SHADER_OTHER
	1185	+ * operation on the first submitted katom after the L2 powering on.
	1186	+ */
	1187	+static void kbase_pm_l2_clear_backend_slot_submit_kctx(struct kbase_device *kbdev)
	1188	+{
	1189	+ int js;
	1190	+
	1191	+ lockdep_assert_held(&kbdev->hwaccess_lock);
	1192	+
	1193	+ /* Clear the slots' last katom submission kctx */
	1194	+ for (js = 0; js < kbdev->gpu_props.num_job_slots; js++)
	1195	+ kbdev->hwaccess.backend.slot_rb[js].last_kctx_tagged = SLOT_RB_NULL_TAG_VAL;
	1196	+}
	1197	+#endif
	1198	+
	1199	+static bool can_power_down_l2(struct kbase_device *kbdev)
	1200	+{
	1201	+#if MALI_USE_CSF
	1202	+ /* Due to the HW issue GPU2019-3878, need to prevent L2 power off
	1203	+ * whilst MMU command is in progress.
	1204	+ * Also defer the power-down if MMU is in process of page migration.
	1205	+ */
	1206	+ return !kbdev->mmu_hw_operation_in_progress && !kbdev->mmu_page_migrate_in_progress;
	1207	+#else
	1208	+ return !kbdev->mmu_page_migrate_in_progress;
	1209	+#endif
	1210	+}
	1211	+
	1212	+static bool can_power_up_l2(struct kbase_device *kbdev)
	1213	+{
	1214	+ lockdep_assert_held(&kbdev->hwaccess_lock);
	1215	+
	1216	+ /* Avoiding l2 transition if MMU is undergoing page migration */
	1217	+ return !kbdev->mmu_page_migrate_in_progress;
	1218	+}
	1219	+
	1220	+static bool need_tiler_control(struct kbase_device *kbdev)
	1221	+{
	1222	+#if MALI_USE_CSF
	1223	+ if (kbase_pm_no_mcu_core_pwroff(kbdev))
	1224	+ return true;
	1225	+ else
	1226	+ return false;
	1227	+#else
	1228	+ return true;
	1229	+#endif
	1230	+}
	1231	+
855	1232	static int kbase_pm_l2_update_state(struct kbase_device *kbdev)
856	1233	{
857	1234	struct kbase_pm_backend_data *backend = &kbdev->pm.backend;
858	1235	u64 l2_present = kbdev->gpu_props.curr_config.l2_present;
859		-#if !MALI_USE_CSF
860	1236	u64 tiler_present = kbdev->gpu_props.props.raw_props.tiler_present;
861		-#endif
	1237	+ bool l2_power_up_done;
862	1238	enum kbase_l2_core_state prev_state;
863	1239
864	1240	lockdep_assert_held(&kbdev->hwaccess_lock);
..	..	@@ -870,54 +1246,76 @@
870	1246	u64 l2_ready = kbase_pm_get_ready_cores(kbdev,
871	1247	KBASE_PM_CORE_L2);
872	1248
873		-#if !MALI_USE_CSF
874		- u64 tiler_trans = kbase_pm_get_trans_cores(kbdev,
875		- KBASE_PM_CORE_TILER);
876		- u64 tiler_ready = kbase_pm_get_ready_cores(kbdev,
877		- KBASE_PM_CORE_TILER);
878		-#endif
879		-
	1249	+#ifdef CONFIG_MALI_ARBITER_SUPPORT
880	1250	/*
881	1251	* kbase_pm_get_ready_cores and kbase_pm_get_trans_cores
882	1252	* are vulnerable to corruption if gpu is lost
883	1253	*/
884		- if (kbase_is_gpu_removed(kbdev)
885		-#ifdef CONFIG_MALI_ARBITER_SUPPORT
886		- \|\| kbase_pm_is_gpu_lost(kbdev)) {
887		-#else
888		- ) {
889		-#endif
	1254	+ if (kbase_is_gpu_removed(kbdev) \|\| kbase_pm_is_gpu_lost(kbdev)) {
890	1255	backend->shaders_state =
891	1256	KBASE_SHADERS_OFF_CORESTACK_OFF;
892		- backend->l2_state = KBASE_L2_OFF;
893		- dev_dbg(kbdev->dev, "GPU lost has occurred - L2 off\n");
	1257	+ backend->hwcnt_desired = false;
	1258	+ if (!backend->hwcnt_disabled) {
	1259	+ /* Don't progress until hw counters are disabled
	1260	+ * This may involve waiting for a worker to complete.
	1261	+ * The HW counters backend disable code checks for the
	1262	+ * GPU removed case and will error out without touching
	1263	+ * the hardware. This step is needed to keep the HW
	1264	+ * counters in a consistent state after a GPU lost.
	1265	+ */
	1266	+ backend->l2_state =
	1267	+ KBASE_L2_ON_HWCNT_DISABLE;
	1268	+ KBASE_KTRACE_ADD(kbdev, PM_L2_ON_HWCNT_DISABLE, NULL,
	1269	+ backend->l2_state);
	1270	+ kbase_pm_trigger_hwcnt_disable(kbdev);
	1271	+ }
	1272	+
	1273	+ if (backend->hwcnt_disabled) {
	1274	+ backend->l2_state = KBASE_L2_OFF;
	1275	+ KBASE_KTRACE_ADD(kbdev, PM_L2_OFF, NULL, backend->l2_state);
	1276	+ dev_dbg(kbdev->dev, "GPU lost has occurred - L2 off\n");
	1277	+ }
894	1278	break;
895	1279	}
	1280	+#endif
896	1281
897	1282	/* mask off ready from trans in case transitions finished
898	1283	* between the register reads
899	1284	*/
900	1285	l2_trans &= ~l2_ready;
901		-#if !MALI_USE_CSF
902		- tiler_trans &= ~tiler_ready;
903		-#endif
	1286	+
904	1287	prev_state = backend->l2_state;
905	1288
906	1289	switch (backend->l2_state) {
907	1290	case KBASE_L2_OFF:
908		- if (kbase_pm_is_l2_desired(kbdev)) {
	1291	+ if (kbase_pm_is_l2_desired(kbdev) && can_power_up_l2(kbdev)) {
	1292	+#if MALI_USE_CSF && defined(KBASE_PM_RUNTIME)
	1293	+ /* Enable HW timer of IPA control before
	1294	+ * L2 cache is powered-up.
	1295	+ */
	1296	+ kbase_ipa_control_handle_gpu_sleep_exit(kbdev);
	1297	+#endif
909	1298	/*
910	1299	* Set the desired config for L2 before
911	1300	* powering it on
912	1301	*/
913	1302	kbase_pm_l2_config_override(kbdev);
914		-#if !MALI_USE_CSF
915		- /* L2 is required, power on. Powering on the
916		- * tiler will also power the first L2 cache.
917		- */
918		- kbase_pm_invoke(kbdev, KBASE_PM_CORE_TILER,
919		- tiler_present, ACTION_PWRON);
	1303	+ kbase_pbha_write_settings(kbdev);
920	1304
	1305	+ /* If Host is controlling the power for shader
	1306	+ * cores, then it also needs to control the
	1307	+ * power for Tiler.
	1308	+ * Powering on the tiler will also power the
	1309	+ * L2 cache.
	1310	+ */
	1311	+ if (need_tiler_control(kbdev)) {
	1312	+ kbase_pm_invoke(kbdev, KBASE_PM_CORE_TILER, tiler_present,
	1313	+ ACTION_PWRON);
	1314	+ } else {
	1315	+ kbase_pm_invoke(kbdev, KBASE_PM_CORE_L2, l2_present,
	1316	+ ACTION_PWRON);
	1317	+ }
	1318	+#if !MALI_USE_CSF
921	1319	/* If we have more than one L2 cache then we
922	1320	* must power them on explicitly.
923	1321	*/
..	..	@@ -925,30 +1323,36 @@
925	1323	kbase_pm_invoke(kbdev, KBASE_PM_CORE_L2,
926	1324	l2_present & ~1,
927	1325	ACTION_PWRON);
928		-#else
929		- /* With CSF firmware, Host driver doesn't need to
930		- * handle power management with both shader and tiler cores.
931		- * The CSF firmware will power up the cores appropriately.
932		- * So only power the l2 cache explicitly.
933		- */
934		- kbase_pm_invoke(kbdev, KBASE_PM_CORE_L2,
935		- l2_present, ACTION_PWRON);
	1326	+ /* Clear backend slot submission kctx */
	1327	+ kbase_pm_l2_clear_backend_slot_submit_kctx(kbdev);
936	1328	#endif
937	1329	backend->l2_state = KBASE_L2_PEND_ON;
938	1330	}
939	1331	break;
940	1332
941	1333	case KBASE_L2_PEND_ON:
942		-#if !MALI_USE_CSF
943		- if (!l2_trans && l2_ready == l2_present && !tiler_trans
944		- && tiler_ready == tiler_present) {
945		- KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_TILER, NULL,
946		- tiler_ready);
947		-#else
	1334	+ l2_power_up_done = false;
948	1335	if (!l2_trans && l2_ready == l2_present) {
949		- KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_L2, NULL,
950		- l2_ready);
951		-#endif
	1336	+ if (need_tiler_control(kbdev)) {
	1337	+ u64 tiler_trans = kbase_pm_get_trans_cores(
	1338	+ kbdev, KBASE_PM_CORE_TILER);
	1339	+ u64 tiler_ready = kbase_pm_get_ready_cores(
	1340	+ kbdev, KBASE_PM_CORE_TILER);
	1341	+ tiler_trans &= ~tiler_ready;
	1342	+
	1343	+ if (!tiler_trans && tiler_ready == tiler_present) {
	1344	+ KBASE_KTRACE_ADD(kbdev,
	1345	+ PM_CORES_CHANGE_AVAILABLE_TILER,
	1346	+ NULL, tiler_ready);
	1347	+ l2_power_up_done = true;
	1348	+ }
	1349	+ } else {
	1350	+ KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_L2, NULL,
	1351	+ l2_ready);
	1352	+ l2_power_up_done = true;
	1353	+ }
	1354	+ }
	1355	+ if (l2_power_up_done) {
952	1356	/*
953	1357	* Ensure snoops are enabled after L2 is powered
954	1358	* up. Note that kbase keeps track of the snoop
..	..	@@ -1027,7 +1431,8 @@
1027	1431	break;
1028	1432	#else
1029	1433	/* Do not power off L2 until the MCU has been stopped */
1030		- if (backend->mcu_state != KBASE_MCU_OFF)
	1434	+ if ((backend->mcu_state != KBASE_MCU_OFF) &&
	1435	+ (backend->mcu_state != KBASE_MCU_IN_SLEEP))
1031	1436	break;
1032	1437	#endif
1033	1438
..	..	@@ -1073,9 +1478,8 @@
1073	1478	}
1074	1479
1075	1480	backend->hwcnt_desired = false;
1076		- if (!backend->hwcnt_disabled) {
	1481	+ if (!backend->hwcnt_disabled)
1077	1482	kbase_pm_trigger_hwcnt_disable(kbdev);
1078		- }
1079	1483	#endif
1080	1484
1081	1485	if (backend->hwcnt_disabled) {
..	..	@@ -1112,27 +1516,31 @@
1112	1516	break;
1113	1517
1114	1518	case KBASE_L2_POWER_DOWN:
1115		- if (!backend->l2_always_on)
1116		- /* Powering off the L2 will also power off the
1117		- * tiler.
1118		- */
1119		- kbase_pm_invoke(kbdev, KBASE_PM_CORE_L2,
1120		- l2_present,
1121		- ACTION_PWROFF);
1122		- else
1123		- /* If L2 cache is powered then we must flush it
1124		- * before we power off the GPU. Normally this
1125		- * would have been handled when the L2 was
1126		- * powered off.
1127		- */
1128		- kbase_gpu_start_cache_clean_nolock(
1129		- kbdev);
	1519	+ if (kbase_pm_is_l2_desired(kbdev))
	1520	+ backend->l2_state = KBASE_L2_PEND_ON;
	1521	+ else if (can_power_down_l2(kbdev)) {
	1522	+ if (!backend->l2_always_on)
	1523	+ /* Powering off the L2 will also power off the
	1524	+ * tiler.
	1525	+ */
	1526	+ kbase_pm_invoke(kbdev, KBASE_PM_CORE_L2,
	1527	+ l2_present,
	1528	+ ACTION_PWROFF);
	1529	+ else
	1530	+ /* If L2 cache is powered then we must flush it
	1531	+ * before we power off the GPU. Normally this
	1532	+ * would have been handled when the L2 was
	1533	+ * powered off.
	1534	+ */
	1535	+ kbase_gpu_start_cache_clean_nolock(
	1536	+ kbdev, GPU_COMMAND_CACHE_CLN_INV_L2);
1130	1537	#if !MALI_USE_CSF
1131		- KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_TILER, NULL, 0u);
	1538	+ KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_TILER, NULL, 0u);
1132	1539	#else
1133		- KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_L2, NULL, 0u);
	1540	+ KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_L2, NULL, 0u);
1134	1541	#endif
1135		- backend->l2_state = KBASE_L2_PEND_OFF;
	1542	+ backend->l2_state = KBASE_L2_PEND_OFF;
	1543	+ }
1136	1544	break;
1137	1545
1138	1546	case KBASE_L2_PEND_OFF:
..	..	@@ -1140,12 +1548,26 @@
1140	1548	/* We only need to check the L2 here - if the L2
1141	1549	* is off then the tiler is definitely also off.
1142	1550	*/
1143		- if (!l2_trans && !l2_ready)
	1551	+ if (!l2_trans && !l2_ready) {
	1552	+#if MALI_USE_CSF && defined(KBASE_PM_RUNTIME)
	1553	+ /* Allow clock gating within the GPU and prevent it
	1554	+ * from being seen as active during sleep.
	1555	+ */
	1556	+ kbase_ipa_control_handle_gpu_sleep_enter(kbdev);
	1557	+#endif
1144	1558	/* L2 is now powered off */
1145	1559	backend->l2_state = KBASE_L2_OFF;
	1560	+ }
1146	1561	} else {
1147		- if (!kbdev->cache_clean_in_progress)
	1562	+ if (!kbdev->cache_clean_in_progress) {
	1563	+#if MALI_USE_CSF && defined(KBASE_PM_RUNTIME)
	1564	+ /* Allow clock gating within the GPU and prevent it
	1565	+ * from being seen as active during sleep.
	1566	+ */
	1567	+ kbase_ipa_control_handle_gpu_sleep_enter(kbdev);
	1568	+#endif
1148	1569	backend->l2_state = KBASE_L2_OFF;
	1570	+ }
1149	1571	}
1150	1572	break;
1151	1573
..	..	@@ -1160,11 +1582,13 @@
1160	1582	backend->l2_state);
1161	1583	}
1162	1584
1163		- if (backend->l2_state != prev_state)
	1585	+ if (backend->l2_state != prev_state) {
1164	1586	dev_dbg(kbdev->dev, "L2 state transition: %s to %s\n",
1165	1587	kbase_l2_core_state_to_string(prev_state),
1166	1588	kbase_l2_core_state_to_string(
1167	1589	backend->l2_state));
	1590	+ kbase_ktrace_log_l2_core_state(kbdev, backend->l2_state);
	1591	+ }
1168	1592
1169	1593	} while (backend->l2_state != prev_state);
1170	1594
..	..	@@ -1503,10 +1927,12 @@
1503	1927	break;
1504	1928
1505	1929	case KBASE_SHADERS_WAIT_FINISHED_CORESTACK_ON:
1506		- shader_poweroff_timer_queue_cancel(kbdev);
	1930	+ if (!backend->partial_shaderoff)
	1931	+ shader_poweroff_timer_queue_cancel(kbdev);
1507	1932
1508	1933	if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TTRX_921)) {
1509		- kbase_gpu_start_cache_clean_nolock(kbdev);
	1934	+ kbase_gpu_start_cache_clean_nolock(
	1935	+ kbdev, GPU_COMMAND_CACHE_CLN_INV_L2);
1510	1936	backend->shaders_state =
1511	1937	KBASE_SHADERS_L2_FLUSHING_CORESTACK_ON;
1512	1938	} else {
..	..	@@ -1608,7 +2034,7 @@
1608	2034
1609	2035	return 0;
1610	2036	}
1611		-#endif
	2037	+#endif /* !MALI_USE_CSF */
1612	2038
1613	2039	static bool kbase_pm_is_in_desired_state_nolock(struct kbase_device *kbdev)
1614	2040	{
..	..	@@ -1616,12 +2042,7 @@
1616	2042
1617	2043	lockdep_assert_held(&kbdev->hwaccess_lock);
1618	2044
1619		- if (kbase_pm_is_l2_desired(kbdev) &&
1620		- kbdev->pm.backend.l2_state != KBASE_L2_ON)
1621		- in_desired_state = false;
1622		- else if (!kbase_pm_is_l2_desired(kbdev) &&
1623		- kbdev->pm.backend.l2_state != KBASE_L2_OFF)
1624		- in_desired_state = false;
	2045	+ in_desired_state = kbase_pm_l2_is_in_desired_state(kbdev);
1625	2046
1626	2047	#if !MALI_USE_CSF
1627	2048	if (kbdev->pm.backend.shaders_desired &&
..	..	@@ -1631,12 +2052,7 @@
1631	2052	kbdev->pm.backend.shaders_state != KBASE_SHADERS_OFF_CORESTACK_OFF)
1632	2053	in_desired_state = false;
1633	2054	#else
1634		- if (kbase_pm_is_mcu_desired(kbdev) &&
1635		- kbdev->pm.backend.mcu_state != KBASE_MCU_ON)
1636		- in_desired_state = false;
1637		- else if (!kbase_pm_is_mcu_desired(kbdev) &&
1638		- kbdev->pm.backend.mcu_state != KBASE_MCU_OFF)
1639		- in_desired_state = false;
	2055	+ in_desired_state &= kbase_pm_mcu_is_in_desired_state(kbdev);
1640	2056	#endif
1641	2057
1642	2058	return in_desired_state;
..	..	@@ -1734,8 +2150,8 @@
1734	2150	if (kbase_pm_mcu_update_state(kbdev))
1735	2151	return;
1736	2152
1737		- if (prev_mcu_state != KBASE_MCU_OFF &&
1738		- kbdev->pm.backend.mcu_state == KBASE_MCU_OFF) {
	2153	+ if (!kbase_pm_is_mcu_inactive(kbdev, prev_mcu_state) &&
	2154	+ kbase_pm_is_mcu_inactive(kbdev, kbdev->pm.backend.mcu_state)) {
1739	2155	if (kbase_pm_l2_update_state(kbdev))
1740	2156	return;
1741	2157	}
..	..	@@ -1803,11 +2219,24 @@
1803	2219	stt->default_ticks = DEFAULT_PM_POWEROFF_TICK_SHADER;
1804	2220	stt->configured_ticks = stt->default_ticks;
1805	2221
	2222	+#if MALI_USE_CSF
	2223	+ kbdev->pm.backend.core_idle_wq = alloc_workqueue("coreoff_wq", WQ_HIGHPRI \| WQ_UNBOUND, 1);
	2224	+ if (!kbdev->pm.backend.core_idle_wq) {
	2225	+ destroy_workqueue(stt->wq);
	2226	+ return -ENOMEM;
	2227	+ }
	2228	+
	2229	+ INIT_WORK(&kbdev->pm.backend.core_idle_work, core_idle_worker);
	2230	+#endif
	2231	+
1806	2232	return 0;
1807	2233	}
1808	2234
1809	2235	void kbase_pm_state_machine_term(struct kbase_device *kbdev)
1810	2236	{
	2237	+#if MALI_USE_CSF
	2238	+ destroy_workqueue(kbdev->pm.backend.core_idle_wq);
	2239	+#endif
1811	2240	hrtimer_cancel(&kbdev->pm.backend.shader_tick_timer.timer);
1812	2241	destroy_workqueue(kbdev->pm.backend.shader_tick_timer.wq);
1813	2242	}
..	..	@@ -1820,6 +2249,7 @@
1820	2249
1821	2250	backend->in_reset = true;
1822	2251	backend->l2_state = KBASE_L2_RESET_WAIT;
	2252	+ KBASE_KTRACE_ADD(kbdev, PM_L2_RESET_WAIT, NULL, backend->l2_state);
1823	2253	#if !MALI_USE_CSF
1824	2254	backend->shaders_state = KBASE_SHADERS_RESET_WAIT;
1825	2255	#else
..	..	@@ -1828,6 +2258,10 @@
1828	2258	*/
1829	2259	if (likely(kbdev->csf.firmware_inited)) {
1830	2260	backend->mcu_state = KBASE_MCU_RESET_WAIT;
	2261	+ KBASE_KTRACE_ADD(kbdev, PM_MCU_RESET_WAIT, NULL, backend->mcu_state);
	2262	+#ifdef KBASE_PM_RUNTIME
	2263	+ backend->exit_gpu_sleep_mode = true;
	2264	+#endif
1831	2265	kbdev->csf.firmware_reload_needed = true;
1832	2266	} else {
1833	2267	WARN_ON(backend->mcu_state != KBASE_MCU_OFF);
..	..	@@ -1865,16 +2299,21 @@
1865	2299	*/
1866	2300	kbase_gpu_cache_clean_wait_complete(kbdev);
1867	2301	backend->in_reset = false;
	2302	+#if MALI_USE_CSF && defined(KBASE_PM_RUNTIME)
	2303	+ backend->gpu_wakeup_override = false;
	2304	+#endif
1868	2305	kbase_pm_update_state(kbdev);
1869	2306
1870	2307	spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
1871	2308	}
1872	2309
1873		-/* Timeout for kbase_pm_wait_for_desired_state when wait_event_killable has
1874		- * aborted due to a fatal signal. If the time spent waiting has exceeded this
1875		- * threshold then there is most likely a hardware issue.
	2310	+#if !MALI_USE_CSF
	2311	+/* Timeout in milliseconds for GPU Power Management to reach the desired
	2312	+ * Shader and L2 state. If the time spent waiting has exceeded this threshold
	2313	+ * then there is most likely a hardware issue.
1876	2314	*/
1877	2315	#define PM_TIMEOUT_MS (5000) /* 5s */
	2316	+#endif
1878	2317
1879	2318	static void kbase_pm_timed_out(struct kbase_device *kbdev)
1880	2319	{
..	..	@@ -1949,19 +2388,21 @@
1949	2388	spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
1950	2389
1951	2390	#if MALI_USE_CSF
1952		- timeout = kbase_csf_timeout_in_jiffies(PM_TIMEOUT_MS);
	2391	+ timeout = kbase_csf_timeout_in_jiffies(kbase_get_timeout_ms(kbdev, CSF_PM_TIMEOUT));
1953	2392	#else
1954	2393	timeout = msecs_to_jiffies(PM_TIMEOUT_MS);
1955	2394	#endif
1956	2395
1957	2396	/* Wait for cores */
1958	2397	#if KERNEL_VERSION(4, 13, 1) <= LINUX_VERSION_CODE
1959		- remaining = wait_event_killable_timeout(
	2398	+ remaining = wait_event_killable_timeout(kbdev->pm.backend.gpu_in_desired_state_wait,
	2399	+ kbase_pm_is_in_desired_state_with_l2_powered(kbdev),
	2400	+ timeout);
1960	2401	#else
1961	2402	remaining = wait_event_timeout(
1962		-#endif
1963	2403	kbdev->pm.backend.gpu_in_desired_state_wait,
1964	2404	kbase_pm_is_in_desired_state_with_l2_powered(kbdev), timeout);
	2405	+#endif
1965	2406
1966	2407	if (!remaining) {
1967	2408	kbase_pm_timed_out(kbdev);
..	..	@@ -1981,7 +2422,7 @@
1981	2422	unsigned long flags;
1982	2423	long remaining;
1983	2424	#if MALI_USE_CSF
1984		- long timeout = kbase_csf_timeout_in_jiffies(PM_TIMEOUT_MS);
	2425	+ long timeout = kbase_csf_timeout_in_jiffies(kbase_get_timeout_ms(kbdev, CSF_PM_TIMEOUT));
1985	2426	#else
1986	2427	long timeout = msecs_to_jiffies(PM_TIMEOUT_MS);
1987	2428	#endif
..	..	@@ -2015,6 +2456,66 @@
2015	2456	return err;
2016	2457	}
2017	2458	KBASE_EXPORT_TEST_API(kbase_pm_wait_for_desired_state);
	2459	+
	2460	+#if MALI_USE_CSF
	2461	+/**
	2462	+ * core_mask_update_done - Check if downscaling of shader cores is done
	2463	+ *
	2464	+ * @kbdev: The kbase device structure for the device.
	2465	+ *
	2466	+ * This function checks if the downscaling of cores is effectively complete.
	2467	+ *
	2468	+ * Return: true if the downscale is done.
	2469	+ */
	2470	+static bool core_mask_update_done(struct kbase_device *kbdev)
	2471	+{
	2472	+ bool update_done = false;
	2473	+ unsigned long flags;
	2474	+
	2475	+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
	2476	+ /* If MCU is in stable ON state then it implies that the downscale
	2477	+ * request had completed.
	2478	+ * If MCU is not active then it implies all cores are off, so can
	2479	+ * consider the downscale request as complete.
	2480	+ */
	2481	+ if ((kbdev->pm.backend.mcu_state == KBASE_MCU_ON) \|\|
	2482	+ kbase_pm_is_mcu_inactive(kbdev, kbdev->pm.backend.mcu_state))
	2483	+ update_done = true;
	2484	+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
	2485	+
	2486	+ return update_done;
	2487	+}
	2488	+
	2489	+int kbase_pm_wait_for_cores_down_scale(struct kbase_device *kbdev)
	2490	+{
	2491	+ long timeout = kbase_csf_timeout_in_jiffies(kbase_get_timeout_ms(kbdev, CSF_PM_TIMEOUT));
	2492	+ long remaining;
	2493	+ int err = 0;
	2494	+
	2495	+ /* Wait for core mask update to complete */
	2496	+#if KERNEL_VERSION(4, 13, 1) <= LINUX_VERSION_CODE
	2497	+ remaining = wait_event_killable_timeout(
	2498	+ kbdev->pm.backend.gpu_in_desired_state_wait,
	2499	+ core_mask_update_done(kbdev), timeout);
	2500	+#else
	2501	+ remaining = wait_event_timeout(
	2502	+ kbdev->pm.backend.gpu_in_desired_state_wait,
	2503	+ core_mask_update_done(kbdev), timeout);
	2504	+#endif
	2505	+
	2506	+ if (!remaining) {
	2507	+ kbase_pm_timed_out(kbdev);
	2508	+ err = -ETIMEDOUT;
	2509	+ } else if (remaining < 0) {
	2510	+ dev_info(
	2511	+ kbdev->dev,
	2512	+ "Wait for cores down scaling got interrupted");
	2513	+ err = (int)remaining;
	2514	+ }
	2515	+
	2516	+ return err;
	2517	+}
	2518	+#endif
2018	2519
2019	2520	void kbase_pm_enable_interrupts(struct kbase_device *kbdev)
2020	2521	{
..	..	@@ -2074,21 +2575,36 @@
2074	2575	KBASE_EXPORT_TEST_API(kbase_pm_disable_interrupts);
2075	2576
2076	2577	#if MALI_USE_CSF
	2578	+/**
	2579	+ * update_user_reg_page_mapping - Update the mapping for USER Register page
	2580	+ *
	2581	+ * @kbdev: The kbase device structure for the device.
	2582	+ *
	2583	+ * This function must be called to unmap the dummy or real page from USER Register page
	2584	+ * mapping whenever GPU is powered up or down. The dummy or real page would get
	2585	+ * appropriately mapped in when Userspace reads the LATEST_FLUSH value.
	2586	+ */
2077	2587	static void update_user_reg_page_mapping(struct kbase_device *kbdev)
2078	2588	{
	2589	+ struct kbase_context kctx, n;
	2590	+
2079	2591	lockdep_assert_held(&kbdev->pm.lock);
2080	2592
2081		- if (kbdev->csf.mali_file_inode) {
2082		- /* This would zap the pte corresponding to the mapping of User
2083		- * register page for all the Kbase contexts.
	2593	+ mutex_lock(&kbdev->csf.reg_lock);
	2594	+ list_for_each_entry_safe(kctx, n, &kbdev->csf.user_reg.list, csf.user_reg.link) {
	2595	+ /* This would zap the PTE corresponding to the mapping of User
	2596	+ * Register page of the kbase context. The mapping will be reestablished
	2597	+ * when the context (user process) needs to access to the page.
2084	2598	*/
2085		- unmap_mapping_range(kbdev->csf.mali_file_inode->i_mapping,
2086		- BASEP_MEM_CSF_USER_REG_PAGE_HANDLE,
2087		- PAGE_SIZE, 1);
	2599	+ unmap_mapping_range(kbdev->csf.user_reg.filp->f_inode->i_mapping,
	2600	+ kctx->csf.user_reg.file_offset << PAGE_SHIFT, PAGE_SIZE, 1);
	2601	+ list_del_init(&kctx->csf.user_reg.link);
	2602	+ dev_dbg(kbdev->dev, "Updated USER Reg page mapping of ctx %d_%d", kctx->tgid,
	2603	+ kctx->id);
2088	2604	}
	2605	+ mutex_unlock(&kbdev->csf.reg_lock);
2089	2606	}
2090	2607	#endif
2091		-
2092	2608
2093	2609	/*
2094	2610	* pmu layout:
..	..	@@ -2098,6 +2614,7 @@
2098	2614	*/
2099	2615	void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume)
2100	2616	{
	2617	+ struct kbase_pm_backend_data *backend = &kbdev->pm.backend;
2101	2618	bool reset_required = is_resume;
2102	2619	unsigned long flags;
2103	2620
..	..	@@ -2115,7 +2632,13 @@
2115	2632	}
2116	2633	#endif
2117	2634
2118		- if (kbdev->pm.backend.gpu_powered) {
	2635	+ if (backend->gpu_powered) {
	2636	+#if MALI_USE_CSF && defined(KBASE_PM_RUNTIME)
	2637	+ if (backend->gpu_idled) {
	2638	+ backend->callback_power_runtime_gpu_active(kbdev);
	2639	+ backend->gpu_idled = false;
	2640	+ }
	2641	+#endif
2119	2642	/* Already turned on */
2120	2643	if (kbdev->poweroff_pending)
2121	2644	kbase_pm_enable_interrupts(kbdev);
..	..	@@ -2128,21 +2651,22 @@
2128	2651
2129	2652	KBASE_KTRACE_ADD(kbdev, PM_GPU_ON, NULL, 0u);
2130	2653
2131		- if (is_resume && kbdev->pm.backend.callback_power_resume) {
2132		- kbdev->pm.backend.callback_power_resume(kbdev);
	2654	+ if (is_resume && backend->callback_power_resume) {
	2655	+ backend->callback_power_resume(kbdev);
2133	2656	return;
2134		- } else if (kbdev->pm.backend.callback_power_on) {
2135		- reset_required = kbdev->pm.backend.callback_power_on(kbdev);
	2657	+ } else if (backend->callback_power_on) {
	2658	+ reset_required = backend->callback_power_on(kbdev);
2136	2659	}
2137	2660
2138	2661	spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
2139		- kbdev->pm.backend.gpu_powered = true;
	2662	+ backend->gpu_powered = true;
2140	2663	spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
2141	2664
2142	2665	#if MALI_USE_CSF
2143	2666	/* GPU has been turned on, can switch to actual register page */
2144	2667	update_user_reg_page_mapping(kbdev);
2145	2668	#endif
	2669	+
2146	2670
2147	2671	if (reset_required) {
2148	2672	/* GPU state was lost, reset GPU to ensure it is in a
..	..	@@ -2194,8 +2718,8 @@
2194	2718
2195	2719	/* Turn on the L2 caches */
2196	2720	spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
2197		- kbdev->pm.backend.gpu_ready = true;
2198		- kbdev->pm.backend.l2_desired = true;
	2721	+ backend->gpu_ready = true;
	2722	+ backend->l2_desired = true;
2199	2723	#if MALI_USE_CSF
2200	2724	if (reset_required) {
2201	2725	/* GPU reset was done after the power on, so send the post
..	..	@@ -2209,6 +2733,16 @@
2209	2733	#endif
2210	2734	kbase_pm_update_state(kbdev);
2211	2735	spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
	2736	+
	2737	+#if MALI_USE_CSF && defined(KBASE_PM_RUNTIME)
	2738	+ /* GPU is now powered up. Invoke the GPU active callback as GPU idle
	2739	+ * callback would have been invoked before the power down.
	2740	+ */
	2741	+ if (backend->gpu_idled) {
	2742	+ backend->callback_power_runtime_gpu_active(kbdev);
	2743	+ backend->gpu_idled = false;
	2744	+ }
	2745	+#endif
2212	2746	}
2213	2747
2214	2748	KBASE_EXPORT_TEST_API(kbase_pm_clock_on);
..	..	@@ -2252,19 +2786,22 @@
2252	2786	kbase_ipa_control_handle_gpu_power_off(kbdev);
2253	2787	#endif
2254	2788
2255		- kbdev->pm.backend.gpu_ready = false;
2256		-
2257		- /* The GPU power may be turned off from this point */
2258		- kbdev->pm.backend.gpu_powered = false;
2259		-
	2789	+ if (kbase_is_gpu_removed(kbdev)
2260	2790	#ifdef CONFIG_MALI_ARBITER_SUPPORT
2261		- if (kbase_pm_is_gpu_lost(kbdev)) {
	2791	+ \|\| kbase_pm_is_gpu_lost(kbdev)) {
	2792	+#else
	2793	+ ) {
	2794	+#endif
2262	2795	/* Ensure we unblock any threads that are stuck waiting
2263	2796	* for the GPU
2264	2797	*/
2265	2798	kbase_gpu_cache_clean_wait_complete(kbdev);
2266	2799	}
2267		-#endif
	2800	+
	2801	+ kbdev->pm.backend.gpu_ready = false;
	2802	+
	2803	+ /* The GPU power may be turned off from this point */
	2804	+ kbdev->pm.backend.gpu_powered = false;
2268	2805
2269	2806	spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
2270	2807
..	..	@@ -2300,9 +2837,9 @@
2300	2837	/**
2301	2838	* kbase_pm_wait_for_reset - Wait for a reset to happen
2302	2839	*
2303		- * Wait for the %RESET_COMPLETED IRQ to occur, then reset the waiting state.
2304		- *
2305	2840	* @kbdev: Kbase device
	2841	+ *
	2842	+ * Wait for the %RESET_COMPLETED IRQ to occur, then reset the waiting state.
2306	2843	*/
2307	2844	static void kbase_pm_wait_for_reset(struct kbase_device *kbdev)
2308	2845	{
..	..	@@ -2431,8 +2968,8 @@
2431	2968	{
2432	2969	struct device_node *np = kbdev->dev->of_node;
2433	2970	const u32 gpu_id = kbdev->gpu_props.props.raw_props.gpu_id;
2434		- const u32 prod_id = (gpu_id & GPU_ID_VERSION_PRODUCT_ID) >>
2435		- GPU_ID_VERSION_PRODUCT_ID_SHIFT;
	2971	+ const u32 prod_id =
	2972	+ (gpu_id & GPU_ID_VERSION_PRODUCT_ID) >> KBASE_GPU_ID_VERSION_PRODUCT_ID_SHIFT;
2436	2973	int error = 0;
2437	2974
2438	2975	kbdev->hw_quirks_gpu = 0;
..	..	@@ -2770,6 +3307,7 @@
2770	3307
2771	3308	/**
2772	3309	* kbase_pm_request_gpu_cycle_counter_do_request - Request cycle counters
	3310	+ * @kbdev: The kbase device structure of the device
2773	3311	*
2774	3312	* Increase the count of cycle counter users and turn the cycle counters on if
2775	3313	* they were previously off
..	..	@@ -2780,8 +3318,6 @@
2780	3318	*
2781	3319	* When this function is called the l2 cache must be on - i.e., the GPU must be
2782	3320	* on.
2783		- *
2784		- * @kbdev: The kbase device structure of the device
2785	3321	*/
2786	3322	static void
2787	3323	kbase_pm_request_gpu_cycle_counter_do_request(struct kbase_device *kbdev)
..	..	@@ -2799,11 +3335,13 @@
2799	3335	/* This might happen after GPU reset.
2800	3336	* Then counter needs to be kicked.
2801	3337	*/
	3338	+#if !IS_ENABLED(CONFIG_MALI_BIFROST_NO_MALI)
2802	3339	if (!(kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_STATUS)) &
2803	3340	GPU_STATUS_CYCLE_COUNT_ACTIVE)) {
2804	3341	kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
2805	3342	GPU_COMMAND_CYCLE_COUNT_START);
2806	3343	}
	3344	+#endif
2807	3345	}
2808	3346
2809	3347	spin_unlock_irqrestore(