~hc/RK356X_SDK_RELEASE.git

..	..	@@ -1,7 +1,7 @@
1	1	// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
2	2	/*
3	3	*
4		- * (C) COPYRIGHT 2018-2021 ARM Limited. All rights reserved.
	4	+ * (C) COPYRIGHT 2018-2023 ARM Limited. All rights reserved.
5	5	*
6	6	* This program is free software and is provided to you under the terms of the
7	7	* GNU General Public License version 2 as published by the Free Software
..	..	@@ -21,40 +21,46 @@
21	21
22	22	#include "mali_kbase.h"
23	23	#include "mali_kbase_csf_firmware_cfg.h"
	24	+#include "mali_kbase_csf_firmware_log.h"
	25	+#include "mali_kbase_csf_firmware_core_dump.h"
24	26	#include "mali_kbase_csf_trace_buffer.h"
25	27	#include "mali_kbase_csf_timeout.h"
26	28	#include "mali_kbase_mem.h"
	29	+#include "mali_kbase_mem_pool_group.h"
27	30	#include "mali_kbase_reset_gpu.h"
28	31	#include "mali_kbase_ctx_sched.h"
29	32	#include "mali_kbase_csf_scheduler.h"
	33	+#include <mali_kbase_hwaccess_time.h>
30	34	#include "device/mali_kbase_device.h"
31	35	#include "backend/gpu/mali_kbase_pm_internal.h"
32	36	#include "tl/mali_kbase_timeline_priv.h"
	37	+#include "tl/mali_kbase_tracepoints.h"
33	38	#include "mali_kbase_csf_tl_reader.h"
34	39	#include "backend/gpu/mali_kbase_clk_rate_trace_mgr.h"
35	40	#include <csf/ipa_control/mali_kbase_csf_ipa_control.h>
36		-
	41	+#include <csf/mali_kbase_csf_registers.h>
37	42	#include <linux/list.h>
38	43	#include <linux/slab.h>
39	44	#include <linux/firmware.h>
40	45	#include <linux/mman.h>
41	46	#include <linux/string.h>
42	47	#include <linux/mutex.h>
	48	+#include <linux/ctype.h>
43	49	#if (KERNEL_VERSION(4, 13, 0) <= LINUX_VERSION_CODE)
44	50	#include <linux/set_memory.h>
45	51	#endif
46	52	#include <mmu/mali_kbase_mmu.h>
47	53	#include <asm/arch_timer.h>
	54	+#include <linux/delay.h>
48	55
49	56	#define MALI_MAX_FIRMWARE_NAME_LEN ((size_t)20)
50		-
51	57
52	58	static char fw_name[MALI_MAX_FIRMWARE_NAME_LEN] = "mali_csffw.bin";
53	59	module_param_string(fw_name, fw_name, sizeof(fw_name), 0644);
54	60	MODULE_PARM_DESC(fw_name, "firmware image");
55	61
56	62	/* The waiting time for firmware to boot */
57		-static unsigned int csf_firmware_boot_timeout_ms = 500;
	63	+static unsigned int csf_firmware_boot_timeout_ms;
58	64	module_param(csf_firmware_boot_timeout_ms, uint, 0444);
59	65	MODULE_PARM_DESC(csf_firmware_boot_timeout_ms,
60	66	"Maximum time to wait for firmware to boot.");
..	..	@@ -72,9 +78,11 @@
72	78	"Enables effective use of a debugger for debugging firmware code.");
73	79	#endif
74	80
75		-#define FIRMWARE_HEADER_MAGIC (0xC3F13A6Eul)
76		-#define FIRMWARE_HEADER_VERSION (0ul)
77		-#define FIRMWARE_HEADER_LENGTH (0x14ul)
	81	+
	82	+#define FIRMWARE_HEADER_MAGIC (0xC3F13A6Eul)
	83	+#define FIRMWARE_HEADER_VERSION_MAJOR (0ul)
	84	+#define FIRMWARE_HEADER_VERSION_MINOR (3ul)
	85	+#define FIRMWARE_HEADER_LENGTH (0x14ul)
78	86
79	87	#define CSF_FIRMWARE_ENTRY_SUPPORTED_FLAGS \
80	88	(CSF_FIRMWARE_ENTRY_READ \| \
..	..	@@ -85,11 +93,13 @@
85	93	CSF_FIRMWARE_ENTRY_ZERO \| \
86	94	CSF_FIRMWARE_ENTRY_CACHE_MODE)
87	95
88		-#define CSF_FIRMWARE_ENTRY_TYPE_INTERFACE (0)
89		-#define CSF_FIRMWARE_ENTRY_TYPE_CONFIGURATION (1)
90		-#define CSF_FIRMWARE_ENTRY_TYPE_FUTF_TEST (2)
91		-#define CSF_FIRMWARE_ENTRY_TYPE_TRACE_BUFFER (3)
92		-#define CSF_FIRMWARE_ENTRY_TYPE_TIMELINE_METADATA (4)
	96	+#define CSF_FIRMWARE_ENTRY_TYPE_INTERFACE (0)
	97	+#define CSF_FIRMWARE_ENTRY_TYPE_CONFIGURATION (1)
	98	+#define CSF_FIRMWARE_ENTRY_TYPE_TRACE_BUFFER (3)
	99	+#define CSF_FIRMWARE_ENTRY_TYPE_TIMELINE_METADATA (4)
	100	+#define CSF_FIRMWARE_ENTRY_TYPE_BUILD_INFO_METADATA (6)
	101	+#define CSF_FIRMWARE_ENTRY_TYPE_FUNC_CALL_LIST (7)
	102	+#define CSF_FIRMWARE_ENTRY_TYPE_CORE_DUMP (9)
93	103
94	104	#define CSF_FIRMWARE_CACHE_MODE_NONE (0ul << 3)
95	105	#define CSF_FIRMWARE_CACHE_MODE_CACHED (1ul << 3)
..	..	@@ -100,11 +110,16 @@
100	110
101	111	#define TL_METADATA_ENTRY_NAME_OFFSET (0x8)
102	112
	113	+#define BUILD_INFO_METADATA_SIZE_OFFSET (0x4)
	114	+#define BUILD_INFO_GIT_SHA_LEN (40U)
	115	+#define BUILD_INFO_GIT_DIRTY_LEN (1U)
	116	+#define BUILD_INFO_GIT_SHA_PATTERN "git_sha: "
	117	+
103	118	#define CSF_MAX_FW_STOP_LOOPS (100000)
104	119
105		-#define CSF_GLB_REQ_CFG_MASK \
106		- (GLB_REQ_CFG_ALLOC_EN_MASK \| GLB_REQ_CFG_PROGRESS_TIMER_MASK \| \
107		- GLB_REQ_CFG_PWROFF_TIMER_MASK)
	120	+#define CSF_GLB_REQ_CFG_MASK \
	121	+ (GLB_REQ_CFG_ALLOC_EN_MASK \| GLB_REQ_CFG_PROGRESS_TIMER_MASK \| \
	122	+ GLB_REQ_CFG_PWROFF_TIMER_MASK \| GLB_REQ_IDLE_ENABLE_MASK)
108	123
109	124	static inline u32 input_page_read(const u32 *const input, const u32 offset)
110	125	{
..	..	@@ -155,8 +170,7 @@
155	170	}
156	171
157	172	/**
158		- * struct firmware_timeline_metadata -
159		- * Timeline metadata item within the MCU firmware
	173	+ * struct firmware_timeline_metadata - Timeline metadata item within the MCU firmware
160	174	*
161	175	* @node: List head linking all timeline metadata to
162	176	* kbase_device:csf.firmware_timeline_metadata.
..	..	@@ -187,11 +201,13 @@
187	201	if (!interface)
188	202	return -EINVAL;
189	203
190		- reg = kbase_alloc_free_region(&kbdev->csf.shared_reg_rbtree, 0,
191		- interface->num_pages, KBASE_REG_ZONE_MCU_SHARED);
	204	+ reg = kbase_alloc_free_region(kbdev, &kbdev->csf.shared_reg_rbtree, 0,
	205	+ interface->num_pages_aligned, KBASE_REG_ZONE_MCU_SHARED);
192	206	if (reg) {
	207	+ mutex_lock(&kbdev->csf.reg_lock);
193	208	ret = kbase_add_va_region_rbtree(kbdev, reg,
194		- interface->virtual, interface->num_pages, 1);
	209	+ interface->virtual, interface->num_pages_aligned, 1);
	210	+ mutex_unlock(&kbdev->csf.reg_lock);
195	211	if (ret)
196	212	kfree(reg);
197	213	else
..	..	@@ -213,10 +229,11 @@
213	229	return (max_loops == 0) ? -1 : 0;
214	230	}
215	231
216		-void kbase_csf_firmware_disable_mcu_wait(struct kbase_device *kbdev)
	232	+void kbase_csf_firmware_disable_mcu(struct kbase_device *kbdev)
217	233	{
218		- if (wait_mcu_status_value(kbdev, MCU_CNTRL_DISABLE) < 0)
219		- dev_err(kbdev->dev, "MCU failed to get disabled");
	234	+ KBASE_TLSTREAM_TL_KBASE_CSFFW_FW_DISABLING(kbdev, kbase_backend_get_cycle_cnt(kbdev));
	235	+
	236	+ kbase_reg_write(kbdev, GPU_CONTROL_REG(MCU_CONTROL), MCU_CNTRL_DISABLE);
220	237	}
221	238
222	239	static void wait_for_firmware_stop(struct kbase_device *kbdev)
..	..	@@ -225,6 +242,13 @@
225	242	/* This error shall go away once MIDJM-2371 is closed */
226	243	dev_err(kbdev->dev, "Firmware failed to stop");
227	244	}
	245	+
	246	+ KBASE_TLSTREAM_TL_KBASE_CSFFW_FW_OFF(kbdev, kbase_backend_get_cycle_cnt(kbdev));
	247	+}
	248	+
	249	+void kbase_csf_firmware_disable_mcu_wait(struct kbase_device *kbdev)
	250	+{
	251	+ wait_for_firmware_stop(kbdev);
228	252	}
229	253
230	254	static void stop_csf_firmware(struct kbase_device *kbdev)
..	..	@@ -237,9 +261,14 @@
237	261
238	262	static void wait_for_firmware_boot(struct kbase_device *kbdev)
239	263	{
240		- const long wait_timeout =
241		- kbase_csf_timeout_in_jiffies(csf_firmware_boot_timeout_ms);
	264	+ long wait_timeout;
242	265	long remaining;
	266	+
	267	+ if (!csf_firmware_boot_timeout_ms)
	268	+ csf_firmware_boot_timeout_ms =
	269	+ kbase_get_timeout_ms(kbdev, CSF_FIRMWARE_BOOT_TIMEOUT);
	270	+
	271	+ wait_timeout = kbase_csf_timeout_in_jiffies(csf_firmware_boot_timeout_ms);
243	272
244	273	/* Firmware will generate a global interface interrupt once booting
245	274	* is complete
..	..	@@ -257,22 +286,51 @@
257	286	{
258	287	kbase_csf_firmware_enable_mcu(kbdev);
259	288
	289	+#if IS_ENABLED(CONFIG_MALI_CORESIGHT)
	290	+ kbase_debug_coresight_csf_state_request(kbdev, KBASE_DEBUG_CORESIGHT_CSF_ENABLED);
	291	+
	292	+ if (!kbase_debug_coresight_csf_state_wait(kbdev, KBASE_DEBUG_CORESIGHT_CSF_ENABLED))
	293	+ dev_err(kbdev->dev, "Timeout waiting for CoreSight to be enabled");
	294	+#endif /* IS_ENABLED(CONFIG_MALI_CORESIGHT) */
	295	+
260	296	wait_for_firmware_boot(kbdev);
261	297	}
262	298
263		-static void wait_ready(struct kbase_device *kbdev)
	299	+/**
	300	+ * wait_ready() - Wait for previously issued MMU command to complete.
	301	+ *
	302	+ * @kbdev: Kbase device to wait for a MMU command to complete.
	303	+ *
	304	+ * Reset GPU if the wait for previously issued command times out.
	305	+ *
	306	+ * Return: 0 on success, error code otherwise.
	307	+ */
	308	+static int wait_ready(struct kbase_device *kbdev)
264	309	{
265		- u32 max_loops = KBASE_AS_INACTIVE_MAX_LOOPS;
266		- u32 val;
	310	+ const ktime_t wait_loop_start = ktime_get_raw();
	311	+ const u32 mmu_as_inactive_wait_time_ms = kbdev->mmu_as_inactive_wait_time_ms;
	312	+ s64 diff;
267	313
268		- val = kbase_reg_read(kbdev, MMU_AS_REG(MCU_AS_NR, AS_STATUS));
	314	+ do {
	315	+ unsigned int i;
269	316
270		- /* Wait for a while for the update command to take effect */
271		- while (--max_loops && (val & AS_STATUS_AS_ACTIVE))
272		- val = kbase_reg_read(kbdev, MMU_AS_REG(MCU_AS_NR, AS_STATUS));
	317	+ for (i = 0; i < 1000; i++) {
	318	+ /* Wait for the MMU status to indicate there is no active command */
	319	+ if (!(kbase_reg_read(kbdev, MMU_AS_REG(MCU_AS_NR, AS_STATUS)) &
	320	+ AS_STATUS_AS_ACTIVE))
	321	+ return 0;
	322	+ }
273	323
274		- if (max_loops == 0)
275		- dev_err(kbdev->dev, "AS_ACTIVE bit stuck, might be caused by slow/unstable GPU clock or possible faulty FPGA connector\n");
	324	+ diff = ktime_to_ms(ktime_sub(ktime_get_raw(), wait_loop_start));
	325	+ } while (diff < mmu_as_inactive_wait_time_ms);
	326	+
	327	+ dev_err(kbdev->dev,
	328	+ "AS_ACTIVE bit stuck for MCU AS. Might be caused by unstable GPU clk/pwr or faulty system");
	329	+
	330	+ if (kbase_prepare_to_reset_gpu_locked(kbdev, RESET_FLAGS_HWC_UNRECOVERABLE_ERROR))
	331	+ kbase_reset_gpu_locked(kbdev);
	332	+
	333	+ return -ETIMEDOUT;
276	334	}
277	335
278	336	static void unload_mmu_tables(struct kbase_device *kbdev)
..	..	@@ -287,7 +345,7 @@
287	345	mutex_unlock(&kbdev->mmu_hw_mutex);
288	346	}
289	347
290		-static void load_mmu_tables(struct kbase_device *kbdev)
	348	+static int load_mmu_tables(struct kbase_device *kbdev)
291	349	{
292	350	unsigned long irq_flags;
293	351
..	..	@@ -298,7 +356,7 @@
298	356	mutex_unlock(&kbdev->mmu_hw_mutex);
299	357
300	358	/* Wait for a while for the update command to take effect */
301		- wait_ready(kbdev);
	359	+ return wait_ready(kbdev);
302	360	}
303	361
304	362	/**
..	..	@@ -405,74 +463,175 @@
405	463	memset(p + copy_len, 0, zi_len);
406	464	}
407	465
408		- kbase_sync_single_for_device(kbdev, kbase_dma_addr(page),
409		- PAGE_SIZE, DMA_TO_DEVICE);
	466	+ kbase_sync_single_for_device(kbdev, kbase_dma_addr_from_tagged(phys[page_num]),
	467	+ PAGE_SIZE, DMA_TO_DEVICE);
410	468	kunmap_atomic(p);
411	469	}
412	470	}
413	471
414		-static int reload_fw_data_sections(struct kbase_device *kbdev)
	472	+static int reload_fw_image(struct kbase_device *kbdev)
415	473	{
416	474	const u32 magic = FIRMWARE_HEADER_MAGIC;
417	475	struct kbase_csf_firmware_interface *interface;
418		- const struct firmware *firmware;
	476	+ struct kbase_csf_mcu_fw *const mcu_fw = &kbdev->csf.fw;
419	477	int ret = 0;
420	478
421		- if (request_firmware(&firmware, fw_name, kbdev->dev) != 0) {
422		- dev_err(kbdev->dev,
423		- "Failed to reload firmware image '%s'\n",
424		- fw_name);
425		- return -ENOENT;
426		- }
427		-
428		- /* Do couple of basic sanity checks */
429		- if (firmware->size < FIRMWARE_HEADER_LENGTH) {
430		- dev_err(kbdev->dev, "Firmware image unexpectedly too small\n");
	479	+ if (WARN_ON(mcu_fw->data == NULL)) {
	480	+ dev_err(kbdev->dev, "Firmware image copy not loaded\n");
431	481	ret = -EINVAL;
432	482	goto out;
433	483	}
434	484
435		- if (memcmp(firmware->data, &magic, sizeof(magic)) != 0) {
	485	+ /* Do a basic sanity check on MAGIC signature */
	486	+ if (memcmp(mcu_fw->data, &magic, sizeof(magic)) != 0) {
436	487	dev_err(kbdev->dev, "Incorrect magic value, firmware image could have been corrupted\n");
437	488	ret = -EINVAL;
438	489	goto out;
439	490	}
440	491
441	492	list_for_each_entry(interface, &kbdev->csf.firmware_interfaces, node) {
442		- /* Skip reload of text & read only data sections */
443		- if ((interface->flags & CSF_FIRMWARE_ENTRY_EXECUTE) \|\|
444		- !(interface->flags & CSF_FIRMWARE_ENTRY_WRITE))
445		- continue;
	493	+ /* Dont skip re-loading any section if full reload was requested */
	494	+ if (!kbdev->csf.firmware_full_reload_needed) {
	495	+ /* Skip reload of text & read only data sections */
	496	+ if ((interface->flags & CSF_FIRMWARE_ENTRY_EXECUTE) \|\|
	497	+ !(interface->flags & CSF_FIRMWARE_ENTRY_WRITE))
	498	+ continue;
	499	+ }
446	500
447		- load_fw_image_section(kbdev, firmware->data, interface->phys,
448		- interface->num_pages, interface->flags,
449		- interface->data_start, interface->data_end);
	501	+ load_fw_image_section(kbdev, mcu_fw->data, interface->phys, interface->num_pages,
	502	+ interface->flags, interface->data_start, interface->data_end);
450	503	}
451	504
452		- kbase_csf_firmware_reload_trace_buffers_data(kbdev);
	505	+ kbdev->csf.firmware_full_reload_needed = false;
453	506
	507	+ kbase_csf_firmware_reload_trace_buffers_data(kbdev);
454	508	out:
455		- release_firmware(firmware);
456	509	return ret;
457	510	}
458	511
459	512	/**
	513	+ * entry_find_large_page_to_reuse() - Find if the large page of previously parsed
	514	+ * FW interface entry can be reused to store
	515	+ * the contents of new FW interface entry.
	516	+ *
	517	+ * @kbdev: Kbase device structure
	518	+ * @virtual_start: Start of the virtual address range required for an entry allocation
	519	+ * @virtual_end: End of the virtual address range required for an entry allocation
	520	+ * @flags: Firmware entry flags for comparison with the reusable pages found
	521	+ * @phys: Pointer to the array of physical (tagged) addresses making up the new
	522	+ * FW interface entry. It is an output parameter which would be made to
	523	+ * point to an already existing array allocated for the previously parsed
	524	+ * FW interface entry using large page(s). If no appropriate entry is
	525	+ * found it is set to NULL.
	526	+ * @pma: Pointer to a protected memory allocation. It is an output parameter
	527	+ * which would be made to the protected memory allocation of a previously
	528	+ * parsed FW interface entry using large page(s) from protected memory.
	529	+ * If no appropriate entry is found it is set to NULL.
	530	+ * @num_pages: Number of pages requested.
	531	+ * @num_pages_aligned: This is an output parameter used to carry the number of 4KB pages
	532	+ * within the 2MB pages aligned allocation.
	533	+ * @is_small_page: This is an output flag used to select between the small and large page
	534	+ * to be used for the FW entry allocation.
	535	+ *
	536	+ * Go through all the already initialized interfaces and find if a previously
	537	+ * allocated large page can be used to store contents of new FW interface entry.
	538	+ *
	539	+ * Return: true if a large page can be reused, false otherwise.
	540	+ */
	541	+static inline bool entry_find_large_page_to_reuse(struct kbase_device *kbdev,
	542	+ const u32 virtual_start, const u32 virtual_end,
	543	+ const u32 flags, struct tagged_addr **phys,
	544	+ struct protected_memory_allocation ***pma,
	545	+ u32 num_pages, u32 *num_pages_aligned,
	546	+ bool *is_small_page)
	547	+{
	548	+ struct kbase_csf_firmware_interface *interface = NULL;
	549	+ struct kbase_csf_firmware_interface *target_interface = NULL;
	550	+ u32 virtual_diff_min = U32_MAX;
	551	+ bool reuse_large_page = false;
	552	+
	553	+ CSTD_UNUSED(interface);
	554	+ CSTD_UNUSED(target_interface);
	555	+ CSTD_UNUSED(virtual_diff_min);
	556	+
	557	+ *num_pages_aligned = num_pages;
	558	+ *is_small_page = true;
	559	+ *phys = NULL;
	560	+ *pma = NULL;
	561	+
	562	+
	563	+ /* If the section starts at 2MB aligned boundary,
	564	+ * then use 2MB page(s) for it.
	565	+ */
	566	+ if (!(virtual_start & (SZ_2M - 1))) {
	567	+ *num_pages_aligned =
	568	+ round_up(*num_pages_aligned, NUM_4K_PAGES_IN_2MB_PAGE);
	569	+ *is_small_page = false;
	570	+ goto out;
	571	+ }
	572	+
	573	+ /* If the section doesn't lie within the same 2MB aligned boundary,
	574	+ * then use 4KB pages as it would be complicated to use a 2MB page
	575	+ * for such section.
	576	+ */
	577	+ if ((virtual_start & ~(SZ_2M - 1)) != (virtual_end & ~(SZ_2M - 1)))
	578	+ goto out;
	579	+
	580	+ /* Find the nearest 2MB aligned section which comes before the current
	581	+ * section.
	582	+ */
	583	+ list_for_each_entry(interface, &kbdev->csf.firmware_interfaces, node) {
	584	+ const u32 virtual_diff = virtual_start - interface->virtual;
	585	+
	586	+ if (interface->virtual > virtual_end)
	587	+ continue;
	588	+
	589	+ if (interface->virtual & (SZ_2M - 1))
	590	+ continue;
	591	+
	592	+ if ((virtual_diff < virtual_diff_min) && (interface->flags == flags)) {
	593	+ target_interface = interface;
	594	+ virtual_diff_min = virtual_diff;
	595	+ }
	596	+ }
	597	+
	598	+ if (target_interface) {
	599	+ const u32 page_index = virtual_diff_min >> PAGE_SHIFT;
	600	+
	601	+ if (page_index >= target_interface->num_pages_aligned)
	602	+ goto out;
	603	+
	604	+ if (target_interface->phys)
	605	+ *phys = &target_interface->phys[page_index];
	606	+
	607	+ if (target_interface->pma)
	608	+ *pma = &target_interface->pma[page_index / NUM_4K_PAGES_IN_2MB_PAGE];
	609	+
	610	+ *is_small_page = false;
	611	+ reuse_large_page = true;
	612	+ }
	613	+
	614	+out:
	615	+ return reuse_large_page;
	616	+}
	617	+
	618	+/**
460	619	* parse_memory_setup_entry() - Process an "interface memory setup" section
	620	+ *
	621	+ * @kbdev: Kbase device structure
	622	+ * @fw: The firmware image containing the section
	623	+ * @entry: Pointer to the start of the section
	624	+ * @size: Size (in bytes) of the section
461	625	*
462	626	* Read an "interface memory setup" section from the firmware image and create
463	627	* the necessary memory region including the MMU page tables. If successful
464	628	* the interface will be added to the kbase_device:csf.firmware_interfaces list.
465	629	*
466	630	* Return: 0 if successful, negative error code on failure
467		- *
468		- * @kbdev: Kbase device structure
469		- * @fw: The firmware image containing the section
470		- * @entry: Pointer to the start of the section
471		- * @size: Size (in bytes) of the section
472	631	*/
473	632	static int parse_memory_setup_entry(struct kbase_device *kbdev,
474		- const struct firmware *fw,
475		- const u32 *entry, unsigned int size)
	633	+ const struct kbase_csf_mcu_fw const fw, const u32 entry,
	634	+ unsigned int size)
476	635	{
477	636	int ret = 0;
478	637	const u32 flags = entry[0];
..	..	@@ -481,13 +640,19 @@
481	640	const u32 data_start = entry[3];
482	641	const u32 data_end = entry[4];
483	642	u32 num_pages;
	643	+ u32 num_pages_aligned;
484	644	char *name;
	645	+ void *name_entry;
	646	+ unsigned int name_len;
485	647	struct tagged_addr *phys = NULL;
486	648	struct kbase_csf_firmware_interface *interface = NULL;
487	649	bool allocated_pages = false, protected_mode = false;
488	650	unsigned long mem_flags = 0;
489	651	u32 cache_mode = 0;
490	652	struct protected_memory_allocation **pma = NULL;
	653	+ bool reuse_pages = false;
	654	+ bool is_small_page = true;
	655	+ bool ignore_page_migration = true;
491	656
492	657	if (data_end < data_start) {
493	658	dev_err(kbdev->dev, "Firmware corrupt, data_end < data_start (0x%x<0x%x)\n",
..	..	@@ -522,7 +687,7 @@
522	687	protected_mode = true;
523	688
524	689	if (protected_mode && kbdev->csf.pma_dev == NULL) {
525		- dev_err(kbdev->dev,
	690	+ dev_dbg(kbdev->dev,
526	691	"Protected memory allocator not found, Firmware protected mode entry will not be supported");
527	692	return 0;
528	693	}
..	..	@@ -530,23 +695,38 @@
530	695	num_pages = (virtual_end - virtual_start)
531	696	>> PAGE_SHIFT;
532	697
533		- phys = kmalloc_array(num_pages, sizeof(*phys), GFP_KERNEL);
	698	+ reuse_pages =
	699	+ entry_find_large_page_to_reuse(kbdev, virtual_start, virtual_end, flags, &phys,
	700	+ &pma, num_pages, &num_pages_aligned, &is_small_page);
	701	+ if (!reuse_pages)
	702	+ phys = kmalloc_array(num_pages_aligned, sizeof(*phys), GFP_KERNEL);
	703	+
534	704	if (!phys)
535	705	return -ENOMEM;
536	706
537	707	if (protected_mode) {
538		- pma = kbase_csf_protected_memory_alloc(kbdev, phys, num_pages);
539		-
540		- if (pma == NULL) {
541		- ret = -ENOMEM;
542		- goto out;
	708	+ if (!reuse_pages) {
	709	+ pma = kbase_csf_protected_memory_alloc(
	710	+ kbdev, phys, num_pages_aligned, is_small_page);
543	711	}
	712	+
	713	+ if (!pma)
	714	+ ret = -ENOMEM;
544	715	} else {
545		- ret = kbase_mem_pool_alloc_pages(
546		- &kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW],
547		- num_pages, phys, false);
548		- if (ret < 0)
549		- goto out;
	716	+ if (!reuse_pages) {
	717	+ ret = kbase_mem_pool_alloc_pages(
	718	+ kbase_mem_pool_group_select(kbdev, KBASE_MEM_GROUP_CSF_FW,
	719	+ is_small_page),
	720	+ num_pages_aligned, phys, false, NULL);
	721	+ ignore_page_migration = false;
	722	+ }
	723	+ }
	724	+
	725	+ if (ret < 0) {
	726	+ dev_err(kbdev->dev,
	727	+ "Failed to allocate %u physical pages for the firmware interface entry at VA 0x%x\n",
	728	+ num_pages_aligned, virtual_start);
	729	+ goto out;
550	730	}
551	731
552	732	allocated_pages = true;
..	..	@@ -554,31 +734,42 @@
554	734	data_start, data_end);
555	735
556	736	/* Allocate enough memory for the struct kbase_csf_firmware_interface and
557		- * the name of the interface. An extra byte is allocated to place a
558		- * NUL-terminator in. This should already be included according to the
559		- * specification but here we add it anyway to be robust against a
560		- * corrupt firmware image.
	737	+ * the name of the interface.
561	738	*/
562		- interface = kmalloc(sizeof(*interface) +
563		- size - INTERFACE_ENTRY_NAME_OFFSET + 1, GFP_KERNEL);
	739	+ name_entry = (void *)entry + INTERFACE_ENTRY_NAME_OFFSET;
	740	+ name_len = strnlen(name_entry, size - INTERFACE_ENTRY_NAME_OFFSET);
	741	+ if (size < (INTERFACE_ENTRY_NAME_OFFSET + name_len + 1 + sizeof(u32))) {
	742	+ dev_err(kbdev->dev, "Memory setup entry too short to contain virtual_exe_start");
	743	+ ret = -EINVAL;
	744	+ goto out;
	745	+ }
	746	+
	747	+ interface = kmalloc(sizeof(*interface) + name_len + 1, GFP_KERNEL);
564	748	if (!interface) {
565	749	ret = -ENOMEM;
566	750	goto out;
567	751	}
568	752	name = (void *)(interface + 1);
569		- memcpy(name, entry + (INTERFACE_ENTRY_NAME_OFFSET / sizeof(*entry)),
570		- size - INTERFACE_ENTRY_NAME_OFFSET);
571		- name[size - INTERFACE_ENTRY_NAME_OFFSET] = 0;
	753	+ memcpy(name, name_entry, name_len);
	754	+ name[name_len] = 0;
572	755
573	756	interface->name = name;
574	757	interface->phys = phys;
	758	+ interface->reuse_pages = reuse_pages;
	759	+ interface->is_small_page = is_small_page;
575	760	interface->num_pages = num_pages;
	761	+ interface->num_pages_aligned = num_pages_aligned;
576	762	interface->virtual = virtual_start;
577	763	interface->kernel_map = NULL;
578	764	interface->flags = flags;
579	765	interface->data_start = data_start;
580	766	interface->data_end = data_end;
581	767	interface->pma = pma;
	768	+
	769	+ /* Discover the virtual execution address field after the end of the name
	770	+ * field taking into account the NULL-termination character.
	771	+ */
	772	+ interface->virtual_exe_start = ((u32 )(name_entry + name_len + 1));
582	773
583	774	mem_flags = convert_mem_flags(kbdev, flags, &cache_mode);
584	775
..	..	@@ -633,15 +824,19 @@
633	824
634	825	list_add(&interface->node, &kbdev->csf.firmware_interfaces);
635	826
636		- ret = kbase_mmu_insert_pages_no_flush(kbdev, &kbdev->csf.mcu_mmu,
637		- virtual_start >> PAGE_SHIFT, phys, num_pages, mem_flags,
638		- KBASE_MEM_GROUP_CSF_FW);
	827	+ if (!reuse_pages) {
	828	+ ret = kbase_mmu_insert_pages_no_flush(kbdev, &kbdev->csf.mcu_mmu,
	829	+ virtual_start >> PAGE_SHIFT, phys,
	830	+ num_pages_aligned, mem_flags,
	831	+ KBASE_MEM_GROUP_CSF_FW, NULL, NULL,
	832	+ ignore_page_migration);
639	833
640		- if (ret != 0) {
641		- dev_err(kbdev->dev, "Failed to insert firmware pages\n");
642		- /* The interface has been added to the list, so cleanup will
643		- * be handled by firmware unloading
644		- */
	834	+ if (ret != 0) {
	835	+ dev_err(kbdev->dev, "Failed to insert firmware pages\n");
	836	+ /* The interface has been added to the list, so cleanup will
	837	+ * be handled by firmware unloading
	838	+ */
	839	+ }
645	840	}
646	841
647	842	dev_dbg(kbdev->dev, "Processed section '%s'", name);
..	..	@@ -650,16 +845,22 @@
650	845
651	846	out:
652	847	if (allocated_pages) {
653		- if (protected_mode) {
654		- kbase_csf_protected_memory_free(kbdev, pma, num_pages);
655		- } else {
656		- kbase_mem_pool_free_pages(
657		- &kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW],
658		- num_pages, phys, false, false);
	848	+ if (!reuse_pages) {
	849	+ if (protected_mode) {
	850	+ kbase_csf_protected_memory_free(
	851	+ kbdev, pma, num_pages_aligned, is_small_page);
	852	+ } else {
	853	+ kbase_mem_pool_free_pages(
	854	+ kbase_mem_pool_group_select(
	855	+ kbdev, KBASE_MEM_GROUP_CSF_FW, is_small_page),
	856	+ num_pages_aligned, phys, false, false);
	857	+ }
659	858	}
660	859	}
661	860
662		- kfree(phys);
	861	+ if (!reuse_pages)
	862	+ kfree(phys);
	863	+
663	864	kfree(interface);
664	865	return ret;
665	866	}
..	..	@@ -675,7 +876,8 @@
675	876	* @size: Size (in bytes) of the section
676	877	*/
677	878	static int parse_timeline_metadata_entry(struct kbase_device *kbdev,
678		- const struct firmware fw, const u32 entry, unsigned int size)
	879	+ const struct kbase_csf_mcu_fw const fw, const u32 entry,
	880	+ unsigned int size)
679	881	{
680	882	const u32 data_start = entry[0];
681	883	const u32 data_size = entry[1];
..	..	@@ -718,7 +920,63 @@
718	920	}
719	921
720	922	/**
	923	+ * parse_build_info_metadata_entry() - Process a "build info metadata" section
	924	+ * @kbdev: Kbase device structure
	925	+ * @fw: Firmware image containing the section
	926	+ * @entry: Pointer to the section
	927	+ * @size: Size (in bytes) of the section
	928	+ *
	929	+ * This prints the git SHA of the firmware on frimware load.
	930	+ *
	931	+ * Return: 0 if successful, negative error code on failure
	932	+ */
	933	+static int parse_build_info_metadata_entry(struct kbase_device *kbdev,
	934	+ const struct kbase_csf_mcu_fw *const fw,
	935	+ const u32 *entry, unsigned int size)
	936	+{
	937	+ const u32 meta_start_addr = entry[0];
	938	+ char *ptr = NULL;
	939	+ size_t sha_pattern_len = strlen(BUILD_INFO_GIT_SHA_PATTERN);
	940	+
	941	+ /* Only print git SHA to avoid releasing sensitive information */
	942	+ ptr = strstr(fw->data + meta_start_addr, BUILD_INFO_GIT_SHA_PATTERN);
	943	+ /* Check that we won't overrun the found string */
	944	+ if (ptr &&
	945	+ strlen(ptr) >= BUILD_INFO_GIT_SHA_LEN + BUILD_INFO_GIT_DIRTY_LEN + sha_pattern_len) {
	946	+ char git_sha[BUILD_INFO_GIT_SHA_LEN + BUILD_INFO_GIT_DIRTY_LEN + 1];
	947	+ int i = 0;
	948	+
	949	+ /* Move ptr to start of SHA */
	950	+ ptr += sha_pattern_len;
	951	+ for (i = 0; i < BUILD_INFO_GIT_SHA_LEN; i++) {
	952	+ /* Ensure that the SHA is made up of hex digits */
	953	+ if (!isxdigit(ptr[i]))
	954	+ break;
	955	+
	956	+ git_sha[i] = ptr[i];
	957	+ }
	958	+
	959	+ /* Check if the next char indicates git SHA is dirty */
	960	+ if (ptr[i] == ' ' \|\| ptr[i] == '+') {
	961	+ git_sha[i] = ptr[i];
	962	+ i++;
	963	+ }
	964	+ git_sha[i] = '\0';
	965	+
	966	+ dev_info(kbdev->dev, "Mali firmware git_sha: %s\n", git_sha);
	967	+ } else
	968	+ dev_info(kbdev->dev, "Mali firmware git_sha not found or invalid\n");
	969	+
	970	+ return 0;
	971	+}
	972	+
	973	+/**
721	974	* load_firmware_entry() - Process an entry from a firmware image
	975	+ *
	976	+ * @kbdev: Kbase device
	977	+ * @fw: Firmware image containing the entry
	978	+ * @offset: Byte offset within the image of the entry to load
	979	+ * @header: Header word of the entry
722	980	*
723	981	* Read an entry from a firmware image and do any necessary work (e.g. loading
724	982	* the data into page accessible to the MCU).
..	..	@@ -727,15 +985,9 @@
727	985	* otherwise the function will fail with -EINVAL
728	986	*
729	987	* Return: 0 if successful, negative error code on failure
730		- *
731		- * @kbdev: Kbase device
732		- * @fw: Firmware image containing the entry
733		- * @offset: Byte offset within the image of the entry to load
734		- * @header: Header word of the entry
735	988	*/
736		-static int load_firmware_entry(struct kbase_device *kbdev,
737		- const struct firmware *fw,
738		- u32 offset, u32 header)
	989	+static int load_firmware_entry(struct kbase_device kbdev, const struct kbase_csf_mcu_fw const fw,
	990	+ u32 offset, u32 header)
739	991	{
740	992	const unsigned int type = entry_type(header);
741	993	unsigned int size = entry_size(header);
..	..	@@ -780,18 +1032,6 @@
780	1032	}
781	1033	return kbase_csf_firmware_cfg_option_entry_parse(
782	1034	kbdev, fw, entry, size, updatable);
783		- case CSF_FIRMWARE_ENTRY_TYPE_FUTF_TEST:
784		-#ifndef MALI_KBASE_BUILD
785		- /* FW UTF option */
786		- if (size < 2sizeof(entry)) {
787		- dev_err(kbdev->dev, "FW UTF entry too short (size=%u)\n",
788		- size);
789		- return -EINVAL;
790		- }
791		- return mali_kutf_process_fw_utf_entry(kbdev, fw->data,
792		- fw->size, entry);
793		-#endif
794		- break;
795	1035	case CSF_FIRMWARE_ENTRY_TYPE_TRACE_BUFFER:
796	1036	/* Trace buffer */
797	1037	if (size < TRACE_BUFFER_ENTRY_NAME_OFFSET + sizeof(*entry)) {
..	..	@@ -809,13 +1049,35 @@
809	1049	return -EINVAL;
810	1050	}
811	1051	return parse_timeline_metadata_entry(kbdev, fw, entry, size);
812		- }
813		-
814		- if (!optional) {
815		- dev_err(kbdev->dev,
816		- "Unsupported non-optional entry type %u in firmware\n",
817		- type);
818		- return -EINVAL;
	1052	+ case CSF_FIRMWARE_ENTRY_TYPE_BUILD_INFO_METADATA:
	1053	+ if (size < BUILD_INFO_METADATA_SIZE_OFFSET + sizeof(*entry)) {
	1054	+ dev_err(kbdev->dev, "Build info metadata entry too short (size=%u)\n",
	1055	+ size);
	1056	+ return -EINVAL;
	1057	+ }
	1058	+ return parse_build_info_metadata_entry(kbdev, fw, entry, size);
	1059	+ case CSF_FIRMWARE_ENTRY_TYPE_FUNC_CALL_LIST:
	1060	+ /* Function call list section */
	1061	+ if (size < FUNC_CALL_LIST_ENTRY_NAME_OFFSET + sizeof(*entry)) {
	1062	+ dev_err(kbdev->dev, "Function call list entry too short (size=%u)\n",
	1063	+ size);
	1064	+ return -EINVAL;
	1065	+ }
	1066	+ kbase_csf_firmware_log_parse_logging_call_list_entry(kbdev, entry);
	1067	+ return 0;
	1068	+ case CSF_FIRMWARE_ENTRY_TYPE_CORE_DUMP:
	1069	+ /* Core Dump section */
	1070	+ if (size < CORE_DUMP_ENTRY_START_ADDR_OFFSET + sizeof(*entry)) {
	1071	+ dev_err(kbdev->dev, "FW Core dump entry too short (size=%u)\n", size);
	1072	+ return -EINVAL;
	1073	+ }
	1074	+ return kbase_csf_firmware_core_dump_entry_parse(kbdev, entry);
	1075	+ default:
	1076	+ if (!optional) {
	1077	+ dev_err(kbdev->dev, "Unsupported non-optional entry type %u in firmware\n",
	1078	+ type);
	1079	+ return -EINVAL;
	1080	+ }
819	1081	}
820	1082
821	1083	return 0;
..	..	@@ -994,11 +1256,10 @@
994	1256	iface->group_stride = shared_info[GLB_GROUP_STRIDE/4];
995	1257	iface->prfcnt_size = shared_info[GLB_PRFCNT_SIZE/4];
996	1258
997		- if (iface->version >= kbase_csf_interface_version(1, 1, 0)) {
	1259	+ if (iface->version >= kbase_csf_interface_version(1, 1, 0))
998	1260	iface->instr_features = shared_info[GLB_INSTR_FEATURES / 4];
999		- } else {
	1261	+ else
1000	1262	iface->instr_features = 0;
1001		- }
1002	1263
1003	1264	if ((GROUP_CONTROL_0 +
1004	1265	(unsigned long)iface->group_num * iface->group_stride) >
..	..	@@ -1033,40 +1294,80 @@
1033	1294	return 0;
1034	1295	}
1035	1296
	1297	+static inline void access_firmware_memory_common(struct kbase_device *kbdev,
	1298	+ struct kbase_csf_firmware_interface *interface, u32 offset_bytes,
	1299	+ u32 *value, const bool read)
	1300	+{
	1301	+ u32 page_num = offset_bytes >> PAGE_SHIFT;
	1302	+ u32 offset_in_page = offset_bytes & ~PAGE_MASK;
	1303	+ struct page *target_page = as_page(interface->phys[page_num]);
	1304	+ uintptr_t cpu_addr = (uintptr_t)kmap_atomic(target_page);
	1305	+ u32 addr = (u32 )(cpu_addr + offset_in_page);
	1306	+
	1307	+ if (read) {
	1308	+ kbase_sync_single_for_device(kbdev,
	1309	+ kbase_dma_addr_from_tagged(interface->phys[page_num]) + offset_in_page,
	1310	+ sizeof(u32), DMA_BIDIRECTIONAL);
	1311	+ value = addr;
	1312	+ } else {
	1313	+ addr = value;
	1314	+ kbase_sync_single_for_device(kbdev,
	1315	+ kbase_dma_addr_from_tagged(interface->phys[page_num]) + offset_in_page,
	1316	+ sizeof(u32), DMA_BIDIRECTIONAL);
	1317	+ }
	1318	+
	1319	+ kunmap_atomic((u32 *)cpu_addr);
	1320	+}
	1321	+
1036	1322	static inline void access_firmware_memory(struct kbase_device *kbdev,
1037	1323	u32 gpu_addr, u32 *value, const bool read)
1038	1324	{
1039		- struct kbase_csf_firmware_interface *interface;
	1325	+ struct kbase_csf_firmware_interface interface, access_interface = NULL;
	1326	+ u32 offset_bytes = 0;
1040	1327
1041	1328	list_for_each_entry(interface, &kbdev->csf.firmware_interfaces, node) {
1042	1329	if ((gpu_addr >= interface->virtual) &&
1043	1330	(gpu_addr < interface->virtual + (interface->num_pages << PAGE_SHIFT))) {
1044		- u32 offset_bytes = gpu_addr - interface->virtual;
1045		- u32 page_num = offset_bytes >> PAGE_SHIFT;
1046		- u32 offset_in_page = offset_bytes & ~PAGE_MASK;
1047		- struct page *target_page = as_page(
1048		- interface->phys[page_num]);
1049		- u32 *cpu_addr = kmap_atomic(target_page);
1050		-
1051		- if (read) {
1052		- kbase_sync_single_for_device(kbdev,
1053		- kbase_dma_addr(target_page) + offset_in_page,
1054		- sizeof(u32), DMA_BIDIRECTIONAL);
1055		-
1056		- *value = cpu_addr[offset_in_page >> 2];
1057		- } else {
1058		- cpu_addr[offset_in_page >> 2] = *value;
1059		-
1060		- kbase_sync_single_for_device(kbdev,
1061		- kbase_dma_addr(target_page) + offset_in_page,
1062		- sizeof(u32), DMA_BIDIRECTIONAL);
1063		- }
1064		-
1065		- kunmap_atomic(cpu_addr);
1066		- return;
	1331	+ offset_bytes = gpu_addr - interface->virtual;
	1332	+ access_interface = interface;
	1333	+ break;
1067	1334	}
1068	1335	}
1069		- dev_warn(kbdev->dev, "Invalid GPU VA %x passed\n", gpu_addr);
	1336	+
	1337	+ if (access_interface)
	1338	+ access_firmware_memory_common(kbdev, access_interface, offset_bytes, value, read);
	1339	+ else
	1340	+ dev_warn(kbdev->dev, "Invalid GPU VA %x passed", gpu_addr);
	1341	+}
	1342	+
	1343	+static inline void access_firmware_memory_exe(struct kbase_device *kbdev,
	1344	+ u32 gpu_addr, u32 *value, const bool read)
	1345	+{
	1346	+ struct kbase_csf_firmware_interface interface, access_interface = NULL;
	1347	+ u32 offset_bytes = 0;
	1348	+
	1349	+ list_for_each_entry(interface, &kbdev->csf.firmware_interfaces, node) {
	1350	+ if ((gpu_addr >= interface->virtual_exe_start) &&
	1351	+ (gpu_addr < interface->virtual_exe_start +
	1352	+ (interface->num_pages << PAGE_SHIFT))) {
	1353	+ offset_bytes = gpu_addr - interface->virtual_exe_start;
	1354	+ access_interface = interface;
	1355	+
	1356	+ /* If there's an overlap in execution address range between a moved and a
	1357	+ * non-moved areas, always prefer the moved one. The idea is that FW may
	1358	+ * move sections around during init time, but after the layout is settled,
	1359	+ * any moved sections are going to override non-moved areas at the same
	1360	+ * location.
	1361	+ */
	1362	+ if (interface->virtual_exe_start != interface->virtual)
	1363	+ break;
	1364	+ }
	1365	+ }
	1366	+
	1367	+ if (access_interface)
	1368	+ access_firmware_memory_common(kbdev, access_interface, offset_bytes, value, read);
	1369	+ else
	1370	+ dev_warn(kbdev->dev, "Invalid GPU VA %x passed", gpu_addr);
1070	1371	}
1071	1372
1072	1373	void kbase_csf_read_firmware_memory(struct kbase_device *kbdev,
..	..	@@ -1079,6 +1380,18 @@
1079	1380	u32 gpu_addr, u32 value)
1080	1381	{
1081	1382	access_firmware_memory(kbdev, gpu_addr, &value, false);
	1383	+}
	1384	+
	1385	+void kbase_csf_read_firmware_memory_exe(struct kbase_device *kbdev,
	1386	+ u32 gpu_addr, u32 *value)
	1387	+{
	1388	+ access_firmware_memory_exe(kbdev, gpu_addr, value, true);
	1389	+}
	1390	+
	1391	+void kbase_csf_update_firmware_memory_exe(struct kbase_device *kbdev,
	1392	+ u32 gpu_addr, u32 value)
	1393	+{
	1394	+ access_firmware_memory_exe(kbdev, gpu_addr, &value, false);
1082	1395	}
1083	1396
1084	1397	void kbase_csf_firmware_cs_input(
..	..	@@ -1166,6 +1479,7 @@
1166	1479	dev_dbg(kbdev->dev, "csg output r: reg %08x val %08x\n", offset, val);
1167	1480	return val;
1168	1481	}
	1482	+KBASE_EXPORT_TEST_API(kbase_csf_firmware_csg_output);
1169	1483
1170	1484	void kbase_csf_firmware_global_input(
1171	1485	const struct kbase_csf_global_iface *const iface, const u32 offset,
..	..	@@ -1176,6 +1490,7 @@
1176	1490	dev_dbg(kbdev->dev, "glob input w: reg %08x val %08x\n", offset, value);
1177	1491	input_page_write(iface->input, offset, value);
1178	1492	}
	1493	+KBASE_EXPORT_TEST_API(kbase_csf_firmware_global_input);
1179	1494
1180	1495	void kbase_csf_firmware_global_input_mask(
1181	1496	const struct kbase_csf_global_iface *const iface, const u32 offset,
..	..	@@ -1187,6 +1502,7 @@
1187	1502	offset, value, mask);
1188	1503	input_page_partial_write(iface->input, offset, value, mask);
1189	1504	}
	1505	+KBASE_EXPORT_TEST_API(kbase_csf_firmware_global_input_mask);
1190	1506
1191	1507	u32 kbase_csf_firmware_global_input_read(
1192	1508	const struct kbase_csf_global_iface *const iface, const u32 offset)
..	..	@@ -1207,6 +1523,27 @@
1207	1523	dev_dbg(kbdev->dev, "glob output r: reg %08x val %08x\n", offset, val);
1208	1524	return val;
1209	1525	}
	1526	+KBASE_EXPORT_TEST_API(kbase_csf_firmware_global_output);
	1527	+
	1528	+/**
	1529	+ * csf_doorbell_offset() - Calculate the offset to the CSF host doorbell
	1530	+ * @doorbell_nr: Doorbell number
	1531	+ *
	1532	+ * Return: CSF host register offset for the specified doorbell number.
	1533	+ */
	1534	+static u32 csf_doorbell_offset(int doorbell_nr)
	1535	+{
	1536	+ WARN_ON(doorbell_nr < 0);
	1537	+ WARN_ON(doorbell_nr >= CSF_NUM_DOORBELL);
	1538	+
	1539	+ return CSF_HW_DOORBELL_PAGE_OFFSET + (doorbell_nr * CSF_HW_DOORBELL_PAGE_SIZE);
	1540	+}
	1541	+
	1542	+void kbase_csf_ring_doorbell(struct kbase_device *kbdev, int doorbell_nr)
	1543	+{
	1544	+ kbase_reg_write(kbdev, csf_doorbell_offset(doorbell_nr), (u32)1);
	1545	+}
	1546	+EXPORT_SYMBOL(kbase_csf_ring_doorbell);
1210	1547
1211	1548	/**
1212	1549	* handle_internal_firmware_fatal - Handler for CS internal firmware fault.
..	..	@@ -1292,11 +1629,10 @@
1292	1629	return complete;
1293	1630	}
1294	1631
1295		-static int wait_for_global_request(struct kbase_device *const kbdev,
1296		- u32 const req_mask)
	1632	+static int wait_for_global_request_with_timeout(struct kbase_device *const kbdev,
	1633	+ u32 const req_mask, unsigned int timeout_ms)
1297	1634	{
1298		- const long wait_timeout =
1299		- kbase_csf_timeout_in_jiffies(kbdev->csf.fw_timeout_ms);
	1635	+ const long wait_timeout = kbase_csf_timeout_in_jiffies(timeout_ms);
1300	1636	long remaining;
1301	1637	int err = 0;
1302	1638
..	..	@@ -1305,12 +1641,19 @@
1305	1641	wait_timeout);
1306	1642
1307	1643	if (!remaining) {
1308		- dev_warn(kbdev->dev, "Timed out waiting for global request %x to complete",
1309		- req_mask);
	1644	+ dev_warn(kbdev->dev,
	1645	+ "[%llu] Timeout (%d ms) waiting for global request %x to complete",
	1646	+ kbase_backend_get_cycle_cnt(kbdev), timeout_ms, req_mask);
1310	1647	err = -ETIMEDOUT;
	1648	+
1311	1649	}
1312	1650
1313	1651	return err;
	1652	+}
	1653	+
	1654	+static int wait_for_global_request(struct kbase_device *const kbdev, u32 const req_mask)
	1655	+{
	1656	+ return wait_for_global_request_with_timeout(kbdev, req_mask, kbdev->csf.fw_timeout_ms);
1314	1657	}
1315	1658
1316	1659	static void set_global_request(
..	..	@@ -1371,16 +1714,113 @@
1371	1714	set_global_request(global_iface, GLB_REQ_CFG_PROGRESS_TIMER_MASK);
1372	1715	}
1373	1716
	1717	+static void enable_gpu_idle_timer(struct kbase_device *const kbdev)
	1718	+{
	1719	+ struct kbase_csf_global_iface *global_iface = &kbdev->csf.global_iface;
	1720	+
	1721	+ kbase_csf_scheduler_spin_lock_assert_held(kbdev);
	1722	+
	1723	+ kbase_csf_firmware_global_input(global_iface, GLB_IDLE_TIMER,
	1724	+ kbdev->csf.gpu_idle_dur_count);
	1725	+ kbase_csf_firmware_global_input_mask(global_iface, GLB_REQ, GLB_REQ_REQ_IDLE_ENABLE,
	1726	+ GLB_REQ_IDLE_ENABLE_MASK);
	1727	+ dev_dbg(kbdev->dev, "Enabling GPU idle timer with count-value: 0x%.8x",
	1728	+ kbdev->csf.gpu_idle_dur_count);
	1729	+}
	1730	+
	1731	+static bool global_debug_request_complete(struct kbase_device *const kbdev, u32 const req_mask)
	1732	+{
	1733	+ struct kbase_csf_global_iface *global_iface = &kbdev->csf.global_iface;
	1734	+ bool complete = false;
	1735	+ unsigned long flags;
	1736	+
	1737	+ kbase_csf_scheduler_spin_lock(kbdev, &flags);
	1738	+
	1739	+ if ((kbase_csf_firmware_global_output(global_iface, GLB_DEBUG_ACK) & req_mask) ==
	1740	+ (kbase_csf_firmware_global_input_read(global_iface, GLB_DEBUG_REQ) & req_mask))
	1741	+ complete = true;
	1742	+
	1743	+ kbase_csf_scheduler_spin_unlock(kbdev, flags);
	1744	+
	1745	+ return complete;
	1746	+}
	1747	+
	1748	+static void set_global_debug_request(const struct kbase_csf_global_iface *const global_iface,
	1749	+ u32 const req_mask)
	1750	+{
	1751	+ u32 glb_debug_req;
	1752	+
	1753	+ kbase_csf_scheduler_spin_lock_assert_held(global_iface->kbdev);
	1754	+
	1755	+ glb_debug_req = kbase_csf_firmware_global_output(global_iface, GLB_DEBUG_ACK);
	1756	+ glb_debug_req ^= req_mask;
	1757	+
	1758	+ kbase_csf_firmware_global_input_mask(global_iface, GLB_DEBUG_REQ, glb_debug_req, req_mask);
	1759	+}
	1760	+
	1761	+static void request_fw_core_dump(
	1762	+ const struct kbase_csf_global_iface *const global_iface)
	1763	+{
	1764	+ uint32_t run_mode = GLB_DEBUG_REQ_RUN_MODE_SET(0, GLB_DEBUG_RUN_MODE_TYPE_CORE_DUMP);
	1765	+
	1766	+ set_global_debug_request(global_iface, GLB_DEBUG_REQ_DEBUG_RUN_MASK \| run_mode);
	1767	+
	1768	+ set_global_request(global_iface, GLB_REQ_DEBUG_CSF_REQ_MASK);
	1769	+}
	1770	+
	1771	+int kbase_csf_firmware_req_core_dump(struct kbase_device *const kbdev)
	1772	+{
	1773	+ const struct kbase_csf_global_iface *const global_iface =
	1774	+ &kbdev->csf.global_iface;
	1775	+ unsigned long flags;
	1776	+ int ret;
	1777	+
	1778	+ /* Serialize CORE_DUMP requests. */
	1779	+ mutex_lock(&kbdev->csf.reg_lock);
	1780	+
	1781	+ /* Update GLB_REQ with CORE_DUMP request and make firmware act on it. */
	1782	+ kbase_csf_scheduler_spin_lock(kbdev, &flags);
	1783	+ request_fw_core_dump(global_iface);
	1784	+ kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR);
	1785	+ kbase_csf_scheduler_spin_unlock(kbdev, flags);
	1786	+
	1787	+ /* Wait for firmware to acknowledge completion of the CORE_DUMP request. */
	1788	+ ret = wait_for_global_request(kbdev, GLB_REQ_DEBUG_CSF_REQ_MASK);
	1789	+ if (!ret)
	1790	+ WARN_ON(!global_debug_request_complete(kbdev, GLB_DEBUG_REQ_DEBUG_RUN_MASK));
	1791	+
	1792	+ mutex_unlock(&kbdev->csf.reg_lock);
	1793	+
	1794	+ return ret;
	1795	+}
	1796	+
	1797	+/**
	1798	+ * kbasep_enable_rtu - Enable Ray Tracing Unit on powering up shader core
	1799	+ *
	1800	+ * @kbdev: The kbase device structure of the device
	1801	+ *
	1802	+ * This function needs to be called to enable the Ray Tracing Unit
	1803	+ * by writing SHADER_PWRFEATURES only when host controls shader cores power.
	1804	+ */
	1805	+static void kbasep_enable_rtu(struct kbase_device *kbdev)
	1806	+{
	1807	+ const u32 gpu_id = kbdev->gpu_props.props.raw_props.gpu_id;
	1808	+
	1809	+ if (gpu_id < GPU_ID2_PRODUCT_MAKE(12, 8, 3, 0))
	1810	+ return;
	1811	+
	1812	+ if (kbdev->csf.firmware_hctl_core_pwr)
	1813	+ kbase_reg_write(kbdev, GPU_CONTROL_REG(SHADER_PWRFEATURES), 1);
	1814	+}
	1815	+
1374	1816	static void global_init(struct kbase_device *const kbdev, u64 core_mask)
1375	1817	{
1376		- u32 const ack_irq_mask = GLB_ACK_IRQ_MASK_CFG_ALLOC_EN_MASK \|
1377		- GLB_ACK_IRQ_MASK_PING_MASK \|
1378		- GLB_ACK_IRQ_MASK_CFG_PROGRESS_TIMER_MASK \|
1379		- GLB_ACK_IRQ_MASK_PROTM_ENTER_MASK \|
1380		- GLB_ACK_IRQ_MASK_PROTM_EXIT_MASK \|
1381		- GLB_ACK_IRQ_MASK_FIRMWARE_CONFIG_UPDATE_MASK \|
1382		- GLB_ACK_IRQ_MASK_CFG_PWROFF_TIMER_MASK \|
1383		- GLB_ACK_IRQ_MASK_IDLE_EVENT_MASK;
	1818	+ u32 const ack_irq_mask =
	1819	+ GLB_ACK_IRQ_MASK_CFG_ALLOC_EN_MASK \| GLB_ACK_IRQ_MASK_PING_MASK \|
	1820	+ GLB_ACK_IRQ_MASK_CFG_PROGRESS_TIMER_MASK \| GLB_ACK_IRQ_MASK_PROTM_ENTER_MASK \|
	1821	+ GLB_ACK_IRQ_MASK_PROTM_EXIT_MASK \| GLB_ACK_IRQ_MASK_FIRMWARE_CONFIG_UPDATE_MASK \|
	1822	+ GLB_ACK_IRQ_MASK_CFG_PWROFF_TIMER_MASK \| GLB_ACK_IRQ_MASK_IDLE_EVENT_MASK \|
	1823	+ GLB_REQ_DEBUG_CSF_REQ_MASK \| GLB_ACK_IRQ_MASK_IDLE_ENABLE_MASK;
1384	1824
1385	1825	const struct kbase_csf_global_iface *const global_iface =
1386	1826	&kbdev->csf.global_iface;
..	..	@@ -1388,10 +1828,7 @@
1388	1828
1389	1829	kbase_csf_scheduler_spin_lock(kbdev, &flags);
1390	1830
1391		- /* Set the coherency mode for protected mode execution */
1392		- WARN_ON(kbdev->system_coherency == COHERENCY_ACE);
1393		- kbase_csf_firmware_global_input(global_iface, GLB_PROTM_COHERENCY,
1394		- kbdev->system_coherency);
	1831	+ kbasep_enable_rtu(kbdev);
1395	1832
1396	1833	/* Update shader core allocation enable mask */
1397	1834	enable_endpoints_global(global_iface, core_mask);
..	..	@@ -1399,9 +1836,23 @@
1399	1836
1400	1837	set_timeout_global(global_iface, kbase_csf_timeout_get(kbdev));
1401	1838
	1839	+ /* The GPU idle timer is always enabled for simplicity. Checks will be
	1840	+ * done before scheduling the GPU idle worker to see if it is
	1841	+ * appropriate for the current power policy.
	1842	+ */
	1843	+ enable_gpu_idle_timer(kbdev);
	1844	+
1402	1845	/* Unmask the interrupts */
1403	1846	kbase_csf_firmware_global_input(global_iface,
1404	1847	GLB_ACK_IRQ_MASK, ack_irq_mask);
	1848	+
	1849	+#if IS_ENABLED(CONFIG_MALI_CORESIGHT)
	1850	+ /* Enable FW MCU read/write debug interfaces */
	1851	+ kbase_csf_firmware_global_input_mask(
	1852	+ global_iface, GLB_DEBUG_ACK_IRQ_MASK,
	1853	+ GLB_DEBUG_REQ_FW_AS_READ_MASK \| GLB_DEBUG_REQ_FW_AS_WRITE_MASK,
	1854	+ GLB_DEBUG_REQ_FW_AS_READ_MASK \| GLB_DEBUG_REQ_FW_AS_WRITE_MASK);
	1855	+#endif /* IS_ENABLED(CONFIG_MALI_CORESIGHT) */
1405	1856
1406	1857	kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR);
1407	1858
..	..	@@ -1482,8 +1933,7 @@
1482	1933	}
1483	1934
1484	1935	/**
1485		- * kbase_csf_firmware_reload_worker() -
1486		- * reload the fw image and re-enable the MCU
	1936	+ * kbase_csf_firmware_reload_worker() - reload the fw image and re-enable the MCU
1487	1937	* @work: CSF Work item for reloading the firmware.
1488	1938	*
1489	1939	* This helper function will reload the firmware image and re-enable the MCU.
..	..	@@ -1503,8 +1953,10 @@
1503	1953
1504	1954	dev_info(kbdev->dev, "reloading firmware");
1505	1955
	1956	+ KBASE_TLSTREAM_TL_KBASE_CSFFW_FW_RELOADING(kbdev, kbase_backend_get_cycle_cnt(kbdev));
	1957	+
1506	1958	/* Reload just the data sections from firmware binary image */
1507		- err = reload_fw_data_sections(kbdev);
	1959	+ err = reload_fw_image(kbdev);
1508	1960	if (err)
1509	1961	return;
1510	1962
..	..	@@ -1545,19 +1997,19 @@
1545	1997	if (version != kbdev->csf.global_iface.version)
1546	1998	dev_err(kbdev->dev, "Version check failed in firmware reboot.");
1547	1999
1548		- KBASE_KTRACE_ADD(kbdev, FIRMWARE_REBOOT, NULL, 0u);
	2000	+ KBASE_KTRACE_ADD(kbdev, CSF_FIRMWARE_REBOOT, NULL, 0u);
1549	2001
1550	2002	/* Tell MCU state machine to transit to next state */
1551	2003	kbdev->csf.firmware_reloaded = true;
1552	2004	kbase_pm_update_state(kbdev);
1553	2005	}
1554	2006
1555		-static u32 convert_dur_to_idle_count(struct kbase_device *kbdev, const u32 dur_ms)
	2007	+static u32 convert_dur_to_idle_count(struct kbase_device *kbdev, const u32 dur_us)
1556	2008	{
1557	2009	#define HYSTERESIS_VAL_UNIT_SHIFT (10)
1558	2010	/* Get the cntfreq_el0 value, which drives the SYSTEM_TIMESTAMP */
1559	2011	u64 freq = arch_timer_get_cntfrq();
1560		- u64 dur_val = dur_ms;
	2012	+ u64 dur_val = dur_us;
1561	2013	u32 cnt_val_u32, reg_val_u32;
1562	2014	bool src_system_timestamp = freq > 0;
1563	2015
..	..	@@ -1570,13 +2022,14 @@
1570	2022	dev_warn(kbdev->dev, "No GPU clock, unexpected intregration issue!");
1571	2023	spin_unlock(&kbdev->pm.clk_rtm.lock);
1572	2024
1573		- dev_info(kbdev->dev, "Can't get the timestamp frequency, "
1574		- "use cycle counter format with firmware idle hysteresis!");
	2025	+ dev_info(
	2026	+ kbdev->dev,
	2027	+ "Can't get the timestamp frequency, use cycle counter format with firmware idle hysteresis!");
1575	2028	}
1576	2029
1577		- /* Formula for dur_val = ((dur_ms/1000) * freq_HZ) >> 10) */
	2030	+ /* Formula for dur_val = ((dur_us/1000000) * freq_HZ) >> 10) */
1578	2031	dur_val = (dur_val * freq) >> HYSTERESIS_VAL_UNIT_SHIFT;
1579		- dur_val = div_u64(dur_val, 1000);
	2032	+ dur_val = div_u64(dur_val, 1000000);
1580	2033
1581	2034	/* Interface limits the value field to S32_MAX */
1582	2035	cnt_val_u32 = (dur_val > S32_MAX) ? S32_MAX : (u32)dur_val;
..	..	@@ -1595,7 +2048,14 @@
1595	2048
1596	2049	u32 kbase_csf_firmware_get_gpu_idle_hysteresis_time(struct kbase_device *kbdev)
1597	2050	{
1598		- return kbdev->csf.gpu_idle_hysteresis_ms;
	2051	+ unsigned long flags;
	2052	+ u32 dur;
	2053	+
	2054	+ kbase_csf_scheduler_spin_lock(kbdev, &flags);
	2055	+ dur = kbdev->csf.gpu_idle_hysteresis_us;
	2056	+ kbase_csf_scheduler_spin_unlock(kbdev, flags);
	2057	+
	2058	+ return dur;
1599	2059	}
1600	2060
1601	2061	u32 kbase_csf_firmware_set_gpu_idle_hysteresis_time(struct kbase_device *kbdev, u32 dur)
..	..	@@ -1603,11 +2063,53 @@
1603	2063	unsigned long flags;
1604	2064	const u32 hysteresis_val = convert_dur_to_idle_count(kbdev, dur);
1605	2065
1606		- kbase_csf_scheduler_spin_lock(kbdev, &flags);
1607		- kbdev->csf.gpu_idle_hysteresis_ms = dur;
1608		- kbdev->csf.gpu_idle_dur_count = hysteresis_val;
1609		- kbase_csf_scheduler_spin_unlock(kbdev, flags);
	2066	+ /* The 'fw_load_lock' is taken to synchronize against the deferred
	2067	+ * loading of FW, where the idle timer will be enabled.
	2068	+ */
	2069	+ mutex_lock(&kbdev->fw_load_lock);
	2070	+ if (unlikely(!kbdev->csf.firmware_inited)) {
	2071	+ kbase_csf_scheduler_spin_lock(kbdev, &flags);
	2072	+ kbdev->csf.gpu_idle_hysteresis_us = dur;
	2073	+ kbdev->csf.gpu_idle_dur_count = hysteresis_val;
	2074	+ kbase_csf_scheduler_spin_unlock(kbdev, flags);
	2075	+ mutex_unlock(&kbdev->fw_load_lock);
	2076	+ goto end;
	2077	+ }
	2078	+ mutex_unlock(&kbdev->fw_load_lock);
1610	2079
	2080	+ kbase_csf_scheduler_pm_active(kbdev);
	2081	+ if (kbase_csf_scheduler_wait_mcu_active(kbdev)) {
	2082	+ dev_err(kbdev->dev,
	2083	+ "Unable to activate the MCU, the idle hysteresis value shall remain unchanged");
	2084	+ kbase_csf_scheduler_pm_idle(kbdev);
	2085	+ return kbdev->csf.gpu_idle_dur_count;
	2086	+ }
	2087	+
	2088	+ /* The 'reg_lock' is also taken and is held till the update is not
	2089	+ * complete, to ensure the update of idle timer value by multiple Users
	2090	+ * gets serialized.
	2091	+ */
	2092	+ mutex_lock(&kbdev->csf.reg_lock);
	2093	+ /* The firmware only reads the new idle timer value when the timer is
	2094	+ * disabled.
	2095	+ */
	2096	+ kbase_csf_scheduler_spin_lock(kbdev, &flags);
	2097	+ kbase_csf_firmware_disable_gpu_idle_timer(kbdev);
	2098	+ kbase_csf_scheduler_spin_unlock(kbdev, flags);
	2099	+ /* Ensure that the request has taken effect */
	2100	+ wait_for_global_request(kbdev, GLB_REQ_IDLE_DISABLE_MASK);
	2101	+
	2102	+ kbase_csf_scheduler_spin_lock(kbdev, &flags);
	2103	+ kbdev->csf.gpu_idle_hysteresis_us = dur;
	2104	+ kbdev->csf.gpu_idle_dur_count = hysteresis_val;
	2105	+ kbase_csf_firmware_enable_gpu_idle_timer(kbdev);
	2106	+ kbase_csf_scheduler_spin_unlock(kbdev, flags);
	2107	+ wait_for_global_request(kbdev, GLB_REQ_IDLE_ENABLE_MASK);
	2108	+ mutex_unlock(&kbdev->csf.reg_lock);
	2109	+
	2110	+ kbase_csf_scheduler_pm_idle(kbdev);
	2111	+
	2112	+end:
1611	2113	dev_dbg(kbdev->dev, "CSF set firmware idle hysteresis count-value: 0x%.8x",
1612	2114	hysteresis_val);
1613	2115
..	..	@@ -1616,7 +2118,6 @@
1616	2118
1617	2119	static u32 convert_dur_to_core_pwroff_count(struct kbase_device *kbdev, const u32 dur_us)
1618	2120	{
1619		-#define PWROFF_VAL_UNIT_SHIFT (10)
1620	2121	/* Get the cntfreq_el0 value, which drives the SYSTEM_TIMESTAMP */
1621	2122	u64 freq = arch_timer_get_cntfrq();
1622	2123	u64 dur_val = dur_us;
..	..	@@ -1632,8 +2133,9 @@
1632	2133	dev_warn(kbdev->dev, "No GPU clock, unexpected integration issue!");
1633	2134	spin_unlock(&kbdev->pm.clk_rtm.lock);
1634	2135
1635		- dev_info(kbdev->dev, "Can't get the timestamp frequency, "
1636		- "use cycle counter with MCU Core Poweroff timer!");
	2136	+ dev_info(
	2137	+ kbdev->dev,
	2138	+ "Can't get the timestamp frequency, use cycle counter with MCU shader Core Poweroff timer!");
1637	2139	}
1638	2140
1639	2141	/* Formula for dur_val = ((dur_us/1e6) * freq_HZ) >> 10) */
..	..	@@ -1657,7 +2159,14 @@
1657	2159
1658	2160	u32 kbase_csf_firmware_get_mcu_core_pwroff_time(struct kbase_device *kbdev)
1659	2161	{
1660		- return kbdev->csf.mcu_core_pwroff_dur_us;
	2162	+ u32 pwroff;
	2163	+ unsigned long flags;
	2164	+
	2165	+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
	2166	+ pwroff = kbdev->csf.mcu_core_pwroff_dur_us;
	2167	+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
	2168	+
	2169	+ return pwroff;
1661	2170	}
1662	2171
1663	2172	u32 kbase_csf_firmware_set_mcu_core_pwroff_time(struct kbase_device *kbdev, u32 dur)
..	..	@@ -1670,34 +2179,124 @@
1670	2179	kbdev->csf.mcu_core_pwroff_dur_count = pwroff;
1671	2180	spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
1672	2181
1673		- dev_dbg(kbdev->dev, "MCU Core Poweroff input update: 0x%.8x", pwroff);
	2182	+ dev_dbg(kbdev->dev, "MCU shader Core Poweroff input update: 0x%.8x", pwroff);
1674	2183
1675	2184	return pwroff;
1676	2185	}
1677	2186
	2187	+/**
	2188	+ * kbase_device_csf_iterator_trace_init - Send request to enable iterator
	2189	+ * trace port.
	2190	+ * @kbdev: Kernel base device pointer
	2191	+ *
	2192	+ * Return: 0 on success (or if enable request is not sent), or error
	2193	+ * code -EINVAL on failure of GPU to acknowledge enable request.
	2194	+ */
	2195	+static int kbase_device_csf_iterator_trace_init(struct kbase_device *kbdev)
	2196	+{
	2197	+ /* Enable the iterator trace port if supported by the GPU.
	2198	+ * It requires the GPU to have a nonzero "iter_trace_enable"
	2199	+ * property in the device tree, and the FW must advertise
	2200	+ * this feature in GLB_FEATURES.
	2201	+ */
	2202	+ if (kbdev->pm.backend.gpu_powered) {
	2203	+ /* check device tree for iterator trace enable property */
	2204	+ const void *iter_trace_param = of_get_property(
	2205	+ kbdev->dev->of_node,
	2206	+ "iter_trace_enable", NULL);
	2207	+
	2208	+ const struct kbase_csf_global_iface *iface =
	2209	+ &kbdev->csf.global_iface;
	2210	+
	2211	+ if (iter_trace_param) {
	2212	+ u32 iter_trace_value = be32_to_cpup(iter_trace_param);
	2213	+
	2214	+ if ((iface->features &
	2215	+ GLB_FEATURES_ITER_TRACE_SUPPORTED_MASK) &&
	2216	+ iter_trace_value) {
	2217	+ long ack_timeout;
	2218	+
	2219	+ ack_timeout = kbase_csf_timeout_in_jiffies(
	2220	+ kbase_get_timeout_ms(kbdev, CSF_FIRMWARE_TIMEOUT));
	2221	+
	2222	+ /* write enable request to global input */
	2223	+ kbase_csf_firmware_global_input_mask(
	2224	+ iface, GLB_REQ,
	2225	+ GLB_REQ_ITER_TRACE_ENABLE_MASK,
	2226	+ GLB_REQ_ITER_TRACE_ENABLE_MASK);
	2227	+ /* Ring global doorbell */
	2228	+ kbase_csf_ring_doorbell(kbdev,
	2229	+ CSF_KERNEL_DOORBELL_NR);
	2230	+
	2231	+ ack_timeout = wait_event_timeout(
	2232	+ kbdev->csf.event_wait,
	2233	+ !((kbase_csf_firmware_global_input_read(
	2234	+ iface, GLB_REQ) ^
	2235	+ kbase_csf_firmware_global_output(
	2236	+ iface, GLB_ACK)) &
	2237	+ GLB_REQ_ITER_TRACE_ENABLE_MASK),
	2238	+ ack_timeout);
	2239	+
	2240	+ return ack_timeout ? 0 : -EINVAL;
	2241	+
	2242	+ }
	2243	+ }
	2244	+
	2245	+ }
	2246	+ return 0;
	2247	+}
1678	2248
1679	2249	int kbase_csf_firmware_early_init(struct kbase_device *kbdev)
1680	2250	{
1681	2251	init_waitqueue_head(&kbdev->csf.event_wait);
1682	2252	kbdev->csf.interrupt_received = false;
1683		- kbdev->csf.fw_timeout_ms = CSF_FIRMWARE_TIMEOUT_MS;
	2253	+
	2254	+ kbdev->csf.fw_timeout_ms =
	2255	+ kbase_get_timeout_ms(kbdev, CSF_FIRMWARE_TIMEOUT);
	2256	+
	2257	+ kbdev->csf.mcu_core_pwroff_dur_us = DEFAULT_GLB_PWROFF_TIMEOUT_US;
	2258	+ kbdev->csf.mcu_core_pwroff_dur_count = convert_dur_to_core_pwroff_count(
	2259	+ kbdev, DEFAULT_GLB_PWROFF_TIMEOUT_US);
1684	2260
1685	2261	INIT_LIST_HEAD(&kbdev->csf.firmware_interfaces);
1686	2262	INIT_LIST_HEAD(&kbdev->csf.firmware_config);
1687	2263	INIT_LIST_HEAD(&kbdev->csf.firmware_timeline_metadata);
1688	2264	INIT_LIST_HEAD(&kbdev->csf.firmware_trace_buffers.list);
	2265	+ INIT_LIST_HEAD(&kbdev->csf.user_reg.list);
1689	2266	INIT_WORK(&kbdev->csf.firmware_reload_work,
1690	2267	kbase_csf_firmware_reload_worker);
1691	2268	INIT_WORK(&kbdev->csf.fw_error_work, firmware_error_worker);
1692	2269
1693	2270	mutex_init(&kbdev->csf.reg_lock);
1694	2271
	2272	+ kbdev->csf.fw = (struct kbase_csf_mcu_fw){ .data = NULL };
	2273	+
1695	2274	return 0;
1696	2275	}
1697	2276
1698		-int kbase_csf_firmware_init(struct kbase_device *kbdev)
	2277	+void kbase_csf_firmware_early_term(struct kbase_device *kbdev)
1699	2278	{
1700		- const struct firmware *firmware;
	2279	+ mutex_destroy(&kbdev->csf.reg_lock);
	2280	+}
	2281	+
	2282	+int kbase_csf_firmware_late_init(struct kbase_device *kbdev)
	2283	+{
	2284	+ kbdev->csf.gpu_idle_hysteresis_us = FIRMWARE_IDLE_HYSTERESIS_TIME_USEC;
	2285	+#ifdef KBASE_PM_RUNTIME
	2286	+ if (kbase_pm_gpu_sleep_allowed(kbdev))
	2287	+ kbdev->csf.gpu_idle_hysteresis_us /= FIRMWARE_IDLE_HYSTERESIS_GPU_SLEEP_SCALER;
	2288	+#endif
	2289	+ WARN_ON(!kbdev->csf.gpu_idle_hysteresis_us);
	2290	+ kbdev->csf.gpu_idle_dur_count =
	2291	+ convert_dur_to_idle_count(kbdev, kbdev->csf.gpu_idle_hysteresis_us);
	2292	+
	2293	+ return 0;
	2294	+}
	2295	+
	2296	+int kbase_csf_firmware_load_init(struct kbase_device *kbdev)
	2297	+{
	2298	+ const struct firmware *firmware = NULL;
	2299	+ struct kbase_csf_mcu_fw *const mcu_fw = &kbdev->csf.fw;
1701	2300	const u32 magic = FIRMWARE_HEADER_MAGIC;
1702	2301	u8 version_major, version_minor;
1703	2302	u32 version_hash;
..	..	@@ -1720,19 +2319,11 @@
1720	2319	return ret;
1721	2320	}
1722	2321
1723		- kbdev->csf.gpu_idle_hysteresis_ms = FIRMWARE_IDLE_HYSTERESIS_TIME_MS;
1724		- kbdev->csf.gpu_idle_dur_count = convert_dur_to_idle_count(
1725		- kbdev, FIRMWARE_IDLE_HYSTERESIS_TIME_MS);
1726		-
1727		- kbdev->csf.mcu_core_pwroff_dur_us = DEFAULT_GLB_PWROFF_TIMEOUT_US;
1728		- kbdev->csf.mcu_core_pwroff_dur_count = convert_dur_to_core_pwroff_count(
1729		- kbdev, DEFAULT_GLB_PWROFF_TIMEOUT_US);
1730		-
1731	2322	ret = kbase_mcu_shared_interface_region_tracker_init(kbdev);
1732	2323	if (ret != 0) {
1733	2324	dev_err(kbdev->dev,
1734	2325	"Failed to setup the rb tree for managing shared interface segment\n");
1735		- goto error;
	2326	+ goto err_out;
1736	2327	}
1737	2328
1738	2329	if (request_firmware(&firmware, fw_name, kbdev->dev) != 0) {
..	..	@@ -1740,43 +2331,60 @@
1740	2331	"Failed to load firmware image '%s'\n",
1741	2332	fw_name);
1742	2333	ret = -ENOENT;
1743		- goto error;
	2334	+ } else {
	2335	+ /* Try to save a copy and then release the loaded firmware image */
	2336	+ mcu_fw->size = firmware->size;
	2337	+ mcu_fw->data = vmalloc((unsigned long)mcu_fw->size);
	2338	+
	2339	+ if (mcu_fw->data == NULL) {
	2340	+ ret = -ENOMEM;
	2341	+ } else {
	2342	+ memcpy(mcu_fw->data, firmware->data, mcu_fw->size);
	2343	+ dev_dbg(kbdev->dev, "Firmware image (%zu-bytes) retained in csf.fw\n",
	2344	+ mcu_fw->size);
	2345	+ }
	2346	+
	2347	+ release_firmware(firmware);
1744	2348	}
1745	2349
1746		- if (firmware->size < FIRMWARE_HEADER_LENGTH) {
	2350	+ /* If error in loading or saving the image, branches to error out */
	2351	+ if (ret)
	2352	+ goto err_out;
	2353	+
	2354	+ if (mcu_fw->size < FIRMWARE_HEADER_LENGTH) {
1747	2355	dev_err(kbdev->dev, "Firmware too small\n");
1748	2356	ret = -EINVAL;
1749		- goto error;
	2357	+ goto err_out;
1750	2358	}
1751	2359
1752		- if (memcmp(firmware->data, &magic, sizeof(magic)) != 0) {
	2360	+ if (memcmp(mcu_fw->data, &magic, sizeof(magic)) != 0) {
1753	2361	dev_err(kbdev->dev, "Incorrect firmware magic\n");
1754	2362	ret = -EINVAL;
1755		- goto error;
	2363	+ goto err_out;
1756	2364	}
1757	2365
1758		- version_minor = firmware->data[4];
1759		- version_major = firmware->data[5];
	2366	+ version_minor = mcu_fw->data[4];
	2367	+ version_major = mcu_fw->data[5];
1760	2368
1761		- if (version_major != FIRMWARE_HEADER_VERSION) {
	2369	+ if (version_major != FIRMWARE_HEADER_VERSION_MAJOR \|\|
	2370	+ version_minor != FIRMWARE_HEADER_VERSION_MINOR) {
1762	2371	dev_err(kbdev->dev,
1763	2372	"Firmware header version %d.%d not understood\n",
1764	2373	version_major, version_minor);
1765	2374	ret = -EINVAL;
1766		- goto error;
	2375	+ goto err_out;
1767	2376	}
1768	2377
1769		- memcpy(&version_hash, &firmware->data[8], sizeof(version_hash));
	2378	+ memcpy(&version_hash, &mcu_fw->data[8], sizeof(version_hash));
1770	2379
1771	2380	dev_notice(kbdev->dev, "Loading Mali firmware 0x%x", version_hash);
1772	2381
1773		- memcpy(&entry_end_offset, &firmware->data[0x10],
1774		- sizeof(entry_end_offset));
	2382	+ memcpy(&entry_end_offset, &mcu_fw->data[0x10], sizeof(entry_end_offset));
1775	2383
1776		- if (entry_end_offset > firmware->size) {
	2384	+ if (entry_end_offset > mcu_fw->size) {
1777	2385	dev_err(kbdev->dev, "Firmware image is truncated\n");
1778	2386	ret = -EINVAL;
1779		- goto error;
	2387	+ goto err_out;
1780	2388	}
1781	2389
1782	2390	entry_offset = FIRMWARE_HEADER_LENGTH;
..	..	@@ -1784,15 +2392,14 @@
1784	2392	u32 header;
1785	2393	unsigned int size;
1786	2394
1787		- memcpy(&header, &firmware->data[entry_offset], sizeof(header));
	2395	+ memcpy(&header, &mcu_fw->data[entry_offset], sizeof(header));
1788	2396
1789	2397	size = entry_size(header);
1790	2398
1791		- ret = load_firmware_entry(kbdev, firmware, entry_offset,
1792		- header);
	2399	+ ret = load_firmware_entry(kbdev, mcu_fw, entry_offset, header);
1793	2400	if (ret != 0) {
1794	2401	dev_err(kbdev->dev, "Failed to load firmware image\n");
1795		- goto error;
	2402	+ goto err_out;
1796	2403	}
1797	2404	entry_offset += size;
1798	2405	}
..	..	@@ -1800,72 +2407,84 @@
1800	2407	if (!kbdev->csf.shared_interface) {
1801	2408	dev_err(kbdev->dev, "Shared interface region not found\n");
1802	2409	ret = -EINVAL;
1803		- goto error;
	2410	+ goto err_out;
1804	2411	} else {
1805	2412	ret = setup_shared_iface_static_region(kbdev);
1806	2413	if (ret != 0) {
1807	2414	dev_err(kbdev->dev, "Failed to insert a region for shared iface entry parsed from fw image\n");
1808		- goto error;
	2415	+ goto err_out;
1809	2416	}
1810	2417	}
1811	2418
1812	2419	ret = kbase_csf_firmware_trace_buffers_init(kbdev);
1813	2420	if (ret != 0) {
1814	2421	dev_err(kbdev->dev, "Failed to initialize trace buffers\n");
1815		- goto error;
	2422	+ goto err_out;
1816	2423	}
1817	2424
1818	2425	/* Make sure L2 cache is powered up */
1819	2426	kbase_pm_wait_for_l2_powered(kbdev);
1820	2427
1821	2428	/* Load the MMU tables into the selected address space */
1822		- load_mmu_tables(kbdev);
	2429	+ ret = load_mmu_tables(kbdev);
	2430	+ if (ret != 0)
	2431	+ goto err_out;
1823	2432
1824	2433	boot_csf_firmware(kbdev);
1825	2434
1826	2435	ret = parse_capabilities(kbdev);
1827	2436	if (ret != 0)
1828		- goto error;
	2437	+ goto err_out;
1829	2438
1830	2439	ret = kbase_csf_doorbell_mapping_init(kbdev);
1831	2440	if (ret != 0)
1832		- goto error;
	2441	+ goto err_out;
1833	2442
1834	2443	ret = kbase_csf_scheduler_init(kbdev);
1835	2444	if (ret != 0)
1836		- goto error;
	2445	+ goto err_out;
1837	2446
1838	2447	ret = kbase_csf_setup_dummy_user_reg_page(kbdev);
1839	2448	if (ret != 0)
1840		- goto error;
	2449	+ goto err_out;
1841	2450
1842	2451	ret = kbase_csf_timeout_init(kbdev);
1843	2452	if (ret != 0)
1844		- goto error;
	2453	+ goto err_out;
1845	2454
1846	2455	ret = global_init_on_boot(kbdev);
1847	2456	if (ret != 0)
1848		- goto error;
	2457	+ goto err_out;
1849	2458
1850	2459	ret = kbase_csf_firmware_cfg_init(kbdev);
1851	2460	if (ret != 0)
1852		- goto error;
	2461	+ goto err_out;
1853	2462
	2463	+ ret = kbase_device_csf_iterator_trace_init(kbdev);
	2464	+ if (ret != 0)
	2465	+ goto err_out;
1854	2466
1855		- /* Firmware loaded successfully */
1856		- release_firmware(firmware);
1857		- KBASE_KTRACE_ADD(kbdev, FIRMWARE_BOOT, NULL,
	2467	+ ret = kbase_csf_firmware_log_init(kbdev);
	2468	+ if (ret != 0) {
	2469	+ dev_err(kbdev->dev, "Failed to initialize FW trace (err %d)", ret);
	2470	+ goto err_out;
	2471	+ }
	2472	+
	2473	+ if (kbdev->csf.fw_core_dump.available)
	2474	+ kbase_csf_firmware_core_dump_init(kbdev);
	2475	+
	2476	+ /* Firmware loaded successfully, ret = 0 */
	2477	+ KBASE_KTRACE_ADD(kbdev, CSF_FIRMWARE_BOOT, NULL,
1858	2478	(((u64)version_hash) << 32) \|
1859	2479	(((u64)version_major) << 8) \| version_minor);
1860	2480	return 0;
1861	2481
1862		-error:
1863		- kbase_csf_firmware_term(kbdev);
1864		- release_firmware(firmware);
	2482	+err_out:
	2483	+ kbase_csf_firmware_unload_term(kbdev);
1865	2484	return ret;
1866	2485	}
1867	2486
1868		-void kbase_csf_firmware_term(struct kbase_device *kbdev)
	2487	+void kbase_csf_firmware_unload_term(struct kbase_device *kbdev)
1869	2488	{
1870	2489	unsigned long flags;
1871	2490	int ret = 0;
..	..	@@ -1875,6 +2494,8 @@
1875	2494	ret = kbase_reset_gpu_wait(kbdev);
1876	2495
1877	2496	WARN(ret, "failed to wait for GPU reset");
	2497	+
	2498	+ kbase_csf_firmware_log_term(kbdev);
1878	2499
1879	2500	kbase_csf_firmware_cfg_term(kbdev);
1880	2501
..	..	@@ -1917,17 +2538,25 @@
1917	2538	list_del(&interface->node);
1918	2539
1919	2540	vunmap(interface->kernel_map);
1920		- if (interface->flags & CSF_FIRMWARE_ENTRY_PROTECTED) {
1921		- kbase_csf_protected_memory_free(kbdev, interface->pma,
1922		- interface->num_pages);
1923		- } else {
1924		- kbase_mem_pool_free_pages(
1925		- &kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW],
1926		- interface->num_pages, interface->phys,
1927		- true, false);
	2541	+
	2542	+ if (!interface->reuse_pages) {
	2543	+ if (interface->flags & CSF_FIRMWARE_ENTRY_PROTECTED) {
	2544	+ kbase_csf_protected_memory_free(
	2545	+ kbdev, interface->pma, interface->num_pages_aligned,
	2546	+ interface->is_small_page);
	2547	+ } else {
	2548	+ kbase_mem_pool_free_pages(
	2549	+ kbase_mem_pool_group_select(
	2550	+ kbdev, KBASE_MEM_GROUP_CSF_FW,
	2551	+ interface->is_small_page),
	2552	+ interface->num_pages_aligned,
	2553	+ interface->phys,
	2554	+ true, false);
	2555	+ }
	2556	+
	2557	+ kfree(interface->phys);
1928	2558	}
1929	2559
1930		- kfree(interface->phys);
1931	2560	kfree(interface);
1932	2561	}
1933	2562
..	..	@@ -1943,16 +2572,17 @@
1943	2572	kfree(metadata);
1944	2573	}
1945	2574
1946		-#ifndef MALI_KBASE_BUILD
1947		- mali_kutf_fw_utf_entry_cleanup(kbdev);
1948		-#endif
	2575	+ if (kbdev->csf.fw.data) {
	2576	+ /* Free the copy of the firmware image */
	2577	+ vfree(kbdev->csf.fw.data);
	2578	+ kbdev->csf.fw.data = NULL;
	2579	+ dev_dbg(kbdev->dev, "Free retained image csf.fw (%zu-bytes)\n", kbdev->csf.fw.size);
	2580	+ }
1949	2581
1950	2582	/* This will also free up the region allocated for the shared interface
1951	2583	* entry parsed from the firmware image.
1952	2584	*/
1953	2585	kbase_mcu_shared_interface_region_tracker_term(kbdev);
1954		-
1955		- mutex_destroy(&kbdev->csf.reg_lock);
1956	2586
1957	2587	kbase_mmu_term(kbdev, &kbdev->csf.mcu_mmu);
1958	2588
..	..	@@ -1960,32 +2590,135 @@
1960	2590	kbdev->as_free \|= MCU_AS_BITMASK;
1961	2591	}
1962	2592
	2593	+#if IS_ENABLED(CONFIG_MALI_CORESIGHT)
	2594	+int kbase_csf_firmware_mcu_register_write(struct kbase_device *const kbdev, u32 const reg_addr,
	2595	+ u32 const reg_val)
	2596	+{
	2597	+ struct kbase_csf_global_iface *global_iface = &kbdev->csf.global_iface;
	2598	+ unsigned long flags;
	2599	+ int err;
	2600	+ u32 glb_req;
	2601	+
	2602	+ mutex_lock(&kbdev->csf.reg_lock);
	2603	+ kbase_csf_scheduler_spin_lock(kbdev, &flags);
	2604	+
	2605	+ /* Set the address and value to write */
	2606	+ kbase_csf_firmware_global_input(global_iface, GLB_DEBUG_ARG_IN0, reg_addr);
	2607	+ kbase_csf_firmware_global_input(global_iface, GLB_DEBUG_ARG_IN1, reg_val);
	2608	+
	2609	+ /* Set the Global Debug request for FW MCU write */
	2610	+ glb_req = kbase_csf_firmware_global_output(global_iface, GLB_DEBUG_ACK);
	2611	+ glb_req ^= GLB_DEBUG_REQ_FW_AS_WRITE_MASK;
	2612	+ kbase_csf_firmware_global_input_mask(global_iface, GLB_DEBUG_REQ, glb_req,
	2613	+ GLB_DEBUG_REQ_FW_AS_WRITE_MASK);
	2614	+
	2615	+ set_global_request(global_iface, GLB_REQ_DEBUG_CSF_REQ_MASK);
	2616	+
	2617	+ /* Notify FW about the Global Debug request */
	2618	+ kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR);
	2619	+
	2620	+ kbase_csf_scheduler_spin_unlock(kbdev, flags);
	2621	+
	2622	+ err = wait_for_global_request(kbdev, GLB_REQ_DEBUG_CSF_REQ_MASK);
	2623	+
	2624	+ mutex_unlock(&kbdev->csf.reg_lock);
	2625	+
	2626	+ dev_dbg(kbdev->dev, "w: reg %08x val %08x", reg_addr, reg_val);
	2627	+
	2628	+ return err;
	2629	+}
	2630	+
	2631	+int kbase_csf_firmware_mcu_register_read(struct kbase_device *const kbdev, u32 const reg_addr,
	2632	+ u32 *reg_val)
	2633	+{
	2634	+ struct kbase_csf_global_iface *global_iface = &kbdev->csf.global_iface;
	2635	+ unsigned long flags;
	2636	+ int err;
	2637	+ u32 glb_req;
	2638	+
	2639	+ if (WARN_ON(reg_val == NULL))
	2640	+ return -EINVAL;
	2641	+
	2642	+ mutex_lock(&kbdev->csf.reg_lock);
	2643	+ kbase_csf_scheduler_spin_lock(kbdev, &flags);
	2644	+
	2645	+ /* Set the address to read */
	2646	+ kbase_csf_firmware_global_input(global_iface, GLB_DEBUG_ARG_IN0, reg_addr);
	2647	+
	2648	+ /* Set the Global Debug request for FW MCU read */
	2649	+ glb_req = kbase_csf_firmware_global_output(global_iface, GLB_DEBUG_ACK);
	2650	+ glb_req ^= GLB_DEBUG_REQ_FW_AS_READ_MASK;
	2651	+ kbase_csf_firmware_global_input_mask(global_iface, GLB_DEBUG_REQ, glb_req,
	2652	+ GLB_DEBUG_REQ_FW_AS_READ_MASK);
	2653	+
	2654	+ set_global_request(global_iface, GLB_REQ_DEBUG_CSF_REQ_MASK);
	2655	+
	2656	+ /* Notify FW about the Global Debug request */
	2657	+ kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR);
	2658	+
	2659	+ kbase_csf_scheduler_spin_unlock(kbdev, flags);
	2660	+
	2661	+ err = wait_for_global_request(kbdev, GLB_REQ_DEBUG_CSF_REQ_MASK);
	2662	+
	2663	+ if (!err) {
	2664	+ kbase_csf_scheduler_spin_lock(kbdev, &flags);
	2665	+ *reg_val = kbase_csf_firmware_global_output(global_iface, GLB_DEBUG_ARG_OUT0);
	2666	+ kbase_csf_scheduler_spin_unlock(kbdev, flags);
	2667	+ }
	2668	+
	2669	+ mutex_unlock(&kbdev->csf.reg_lock);
	2670	+
	2671	+ dev_dbg(kbdev->dev, "r: reg %08x val %08x", reg_addr, *reg_val);
	2672	+
	2673	+ return err;
	2674	+}
	2675	+
	2676	+int kbase_csf_firmware_mcu_register_poll(struct kbase_device *const kbdev, u32 const reg_addr,
	2677	+ u32 const val_mask, u32 const reg_val)
	2678	+{
	2679	+ unsigned long remaining = kbase_csf_timeout_in_jiffies(kbdev->csf.fw_timeout_ms) + jiffies;
	2680	+ u32 read_val;
	2681	+
	2682	+ dev_dbg(kbdev->dev, "p: reg %08x val %08x mask %08x", reg_addr, reg_val, val_mask);
	2683	+
	2684	+ while (time_before(jiffies, remaining)) {
	2685	+ int err = kbase_csf_firmware_mcu_register_read(kbdev, reg_addr, &read_val);
	2686	+
	2687	+ if (err) {
	2688	+ dev_err(kbdev->dev,
	2689	+ "Error reading MCU register value (read_val = %u, expect = %u)\n",
	2690	+ read_val, reg_val);
	2691	+ return err;
	2692	+ }
	2693	+
	2694	+ if ((read_val & val_mask) == reg_val)
	2695	+ return 0;
	2696	+ }
	2697	+
	2698	+ dev_err(kbdev->dev,
	2699	+ "Timeout waiting for MCU register value to be set (read_val = %u, expect = %u)\n",
	2700	+ read_val, reg_val);
	2701	+
	2702	+ return -ETIMEDOUT;
	2703	+}
	2704	+#endif /* IS_ENABLED(CONFIG_MALI_CORESIGHT) */
	2705	+
1963	2706	void kbase_csf_firmware_enable_gpu_idle_timer(struct kbase_device *kbdev)
1964	2707	{
1965	2708	struct kbase_csf_global_iface *global_iface = &kbdev->csf.global_iface;
1966		- const u32 glb_req =
1967		- kbase_csf_firmware_global_input_read(global_iface, GLB_REQ);
	2709	+ const u32 glb_req = kbase_csf_firmware_global_input_read(global_iface, GLB_REQ);
1968	2710
1969	2711	kbase_csf_scheduler_spin_lock_assert_held(kbdev);
1970		-
1971	2712	/* The scheduler is assumed to only call the enable when its internal
1972	2713	* state indicates that the idle timer has previously been disabled. So
1973	2714	* on entry the expected field values are:
1974	2715	* 1. GLOBAL_INPUT_BLOCK.GLB_REQ.IDLE_ENABLE: 0
1975	2716	* 2. GLOBAL_OUTPUT_BLOCK.GLB_ACK.IDLE_ENABLE: 0, or, on 1 -> 0
1976	2717	*/
1977		-
1978	2718	if (glb_req & GLB_REQ_IDLE_ENABLE_MASK)
1979	2719	dev_err(kbdev->dev, "Incoherent scheduler state on REQ_IDLE_ENABLE!");
1980	2720
1981		- kbase_csf_firmware_global_input(global_iface, GLB_IDLE_TIMER,
1982		- kbdev->csf.gpu_idle_dur_count);
1983		-
1984		- kbase_csf_firmware_global_input_mask(global_iface, GLB_REQ,
1985		- GLB_REQ_REQ_IDLE_ENABLE, GLB_REQ_IDLE_ENABLE_MASK);
1986		-
1987		- dev_dbg(kbdev->dev, "Enabling GPU idle timer with count-value: 0x%.8x",
1988		- kbdev->csf.gpu_idle_dur_count);
	2721	+ enable_gpu_idle_timer(kbdev);
1989	2722	kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR);
1990	2723	}
1991	2724
..	..	@@ -2015,10 +2748,11 @@
2015	2748	kbase_csf_scheduler_spin_unlock(kbdev, flags);
2016	2749	}
2017	2750
2018		-int kbase_csf_firmware_ping_wait(struct kbase_device *const kbdev)
	2751	+int kbase_csf_firmware_ping_wait(struct kbase_device *const kbdev, unsigned int wait_timeout_ms)
2019	2752	{
2020	2753	kbase_csf_firmware_ping(kbdev);
2021		- return wait_for_global_request(kbdev, GLB_REQ_PING_MASK);
	2754	+
	2755	+ return wait_for_global_request_with_timeout(kbdev, GLB_REQ_PING_MASK, wait_timeout_ms);
2022	2756	}
2023	2757
2024	2758	int kbase_csf_firmware_set_timeout(struct kbase_device *const kbdev,
..	..	@@ -2048,31 +2782,63 @@
2048	2782	void kbase_csf_enter_protected_mode(struct kbase_device *kbdev)
2049	2783	{
2050	2784	struct kbase_csf_global_iface *global_iface = &kbdev->csf.global_iface;
2051		- unsigned long flags;
2052		- int err;
2053	2785
2054		- kbase_csf_scheduler_spin_lock(kbdev, &flags);
	2786	+ KBASE_TLSTREAM_AUX_PROTECTED_ENTER_START(kbdev, kbdev);
	2787	+
	2788	+ kbase_csf_scheduler_spin_lock_assert_held(kbdev);
2055	2789	set_global_request(global_iface, GLB_REQ_PROTM_ENTER_MASK);
2056	2790	dev_dbg(kbdev->dev, "Sending request to enter protected mode");
2057	2791	kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR);
2058		- kbase_csf_scheduler_spin_unlock(kbdev, flags);
	2792	+}
	2793	+
	2794	+int kbase_csf_wait_protected_mode_enter(struct kbase_device *kbdev)
	2795	+{
	2796	+ int err;
	2797	+
	2798	+ lockdep_assert_held(&kbdev->mmu_hw_mutex);
2059	2799
2060	2800	err = wait_for_global_request(kbdev, GLB_REQ_PROTM_ENTER_MASK);
2061	2801
2062	2802	if (!err) {
2063		- unsigned long irq_flags;
	2803	+#define WAIT_TIMEOUT 5000 /* 50ms timeout */
	2804	+#define DELAY_TIME_IN_US 10
	2805	+ const int max_iterations = WAIT_TIMEOUT;
	2806	+ int loop;
2064	2807
2065		- spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
2066		- kbdev->protected_mode = true;
2067		- kbase_ipa_protection_mode_switch_event(kbdev);
2068		- kbase_ipa_control_protm_entered(kbdev);
	2808	+ /* Wait for the GPU to actually enter protected mode */
	2809	+ for (loop = 0; loop < max_iterations; loop++) {
	2810	+ unsigned long flags;
	2811	+ bool pmode_exited;
2069	2812
2070		- kbase_csf_scheduler_spin_lock(kbdev, &irq_flags);
2071		- kbase_hwcnt_backend_csf_protm_entered(&kbdev->hwcnt_gpu_iface);
2072		- kbase_csf_scheduler_spin_unlock(kbdev, irq_flags);
	2813	+ if (kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_STATUS)) &
	2814	+ GPU_STATUS_PROTECTED_MODE_ACTIVE)
	2815	+ break;
2073	2816
2074		- spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
	2817	+ /* Check if GPU already exited the protected mode */
	2818	+ kbase_csf_scheduler_spin_lock(kbdev, &flags);
	2819	+ pmode_exited =
	2820	+ !kbase_csf_scheduler_protected_mode_in_use(kbdev);
	2821	+ kbase_csf_scheduler_spin_unlock(kbdev, flags);
	2822	+ if (pmode_exited)
	2823	+ break;
	2824	+
	2825	+ udelay(DELAY_TIME_IN_US);
	2826	+ }
	2827	+
	2828	+ if (loop == max_iterations) {
	2829	+ dev_err(kbdev->dev, "Timeout for actual pmode entry after PROTM_ENTER ack");
	2830	+ err = -ETIMEDOUT;
	2831	+ }
2075	2832	}
	2833	+
	2834	+ if (unlikely(err)) {
	2835	+ if (kbase_prepare_to_reset_gpu(kbdev, RESET_FLAGS_HWC_UNRECOVERABLE_ERROR))
	2836	+ kbase_reset_gpu(kbdev);
	2837	+ }
	2838	+
	2839	+ KBASE_TLSTREAM_AUX_PROTECTED_ENTER_END(kbdev, kbdev);
	2840	+
	2841	+ return err;
2076	2842	}
2077	2843
2078	2844	void kbase_csf_firmware_trigger_mcu_halt(struct kbase_device *kbdev)
..	..	@@ -2080,12 +2846,53 @@
2080	2846	struct kbase_csf_global_iface *global_iface = &kbdev->csf.global_iface;
2081	2847	unsigned long flags;
2082	2848
	2849	+ KBASE_TLSTREAM_TL_KBASE_CSFFW_FW_REQUEST_HALT(kbdev, kbase_backend_get_cycle_cnt(kbdev));
	2850	+
2083	2851	kbase_csf_scheduler_spin_lock(kbdev, &flags);
	2852	+ /* Validate there are no on-slot groups when sending the
	2853	+ * halt request to firmware.
	2854	+ */
	2855	+ WARN_ON(kbase_csf_scheduler_get_nr_active_csgs_locked(kbdev));
2084	2856	set_global_request(global_iface, GLB_REQ_HALT_MASK);
2085	2857	dev_dbg(kbdev->dev, "Sending request to HALT MCU");
2086	2858	kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR);
2087	2859	kbase_csf_scheduler_spin_unlock(kbdev, flags);
2088	2860	}
	2861	+
	2862	+void kbase_csf_firmware_enable_mcu(struct kbase_device *kbdev)
	2863	+{
	2864	+ KBASE_TLSTREAM_TL_KBASE_CSFFW_FW_ENABLING(kbdev, kbase_backend_get_cycle_cnt(kbdev));
	2865	+
	2866	+ /* Trigger the boot of MCU firmware, Use the AUTO mode as
	2867	+ * otherwise on fast reset, to exit protected mode, MCU will
	2868	+ * not reboot by itself to enter normal mode.
	2869	+ */
	2870	+ kbase_reg_write(kbdev, GPU_CONTROL_REG(MCU_CONTROL), MCU_CNTRL_AUTO);
	2871	+}
	2872	+
	2873	+#ifdef KBASE_PM_RUNTIME
	2874	+void kbase_csf_firmware_trigger_mcu_sleep(struct kbase_device *kbdev)
	2875	+{
	2876	+ struct kbase_csf_global_iface *global_iface = &kbdev->csf.global_iface;
	2877	+ unsigned long flags;
	2878	+
	2879	+ KBASE_TLSTREAM_TL_KBASE_CSFFW_FW_REQUEST_SLEEP(kbdev, kbase_backend_get_cycle_cnt(kbdev));
	2880	+
	2881	+ kbase_csf_scheduler_spin_lock(kbdev, &flags);
	2882	+ set_global_request(global_iface, GLB_REQ_SLEEP_MASK);
	2883	+ dev_dbg(kbdev->dev, "Sending sleep request to MCU");
	2884	+ kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR);
	2885	+ kbase_csf_scheduler_spin_unlock(kbdev, flags);
	2886	+}
	2887	+
	2888	+bool kbase_csf_firmware_is_mcu_in_sleep(struct kbase_device *kbdev)
	2889	+{
	2890	+ lockdep_assert_held(&kbdev->hwaccess_lock);
	2891	+
	2892	+ return (global_request_complete(kbdev, GLB_REQ_SLEEP_MASK) &&
	2893	+ kbase_csf_firmware_mcu_halted(kbdev));
	2894	+}
	2895	+#endif
2089	2896
2090	2897	int kbase_csf_trigger_firmware_config_update(struct kbase_device *kbdev)
2091	2898	{
..	..	@@ -2095,6 +2902,7 @@
2095	2902
2096	2903	/* Ensure GPU is powered-up until we complete config update.*/
2097	2904	kbase_csf_scheduler_pm_active(kbdev);
	2905	+ kbase_csf_scheduler_wait_mcu_active(kbdev);
2098	2906
2099	2907	/* The 'reg_lock' is also taken and is held till the update is
2100	2908	* complete, to ensure the config update gets serialized.
..	..	@@ -2234,7 +3042,7 @@
2234	3042	gpu_map_prot =
2235	3043	KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_NON_CACHEABLE);
2236	3044	cpu_map_prot = pgprot_writecombine(cpu_map_prot);
2237		- };
	3045	+ }
2238	3046
2239	3047	phys = kmalloc_array(num_pages, sizeof(*phys), GFP_KERNEL);
2240	3048	if (!phys)
..	..	@@ -2244,9 +3052,8 @@
2244	3052	if (!page_list)
2245	3053	goto page_list_alloc_error;
2246	3054
2247		- ret = kbase_mem_pool_alloc_pages(
2248		- &kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW],
2249		- num_pages, phys, false);
	3055	+ ret = kbase_mem_pool_alloc_pages(&kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], num_pages,
	3056	+ phys, false, NULL);
2250	3057	if (ret <= 0)
2251	3058	goto phys_mem_pool_alloc_error;
2252	3059
..	..	@@ -2257,8 +3064,8 @@
2257	3064	if (!cpu_addr)
2258	3065	goto vmap_error;
2259	3066
2260		- va_reg = kbase_alloc_free_region(&kbdev->csf.shared_reg_rbtree, 0,
2261		- num_pages, KBASE_REG_ZONE_MCU_SHARED);
	3067	+ va_reg = kbase_alloc_free_region(kbdev, &kbdev->csf.shared_reg_rbtree, 0, num_pages,
	3068	+ KBASE_REG_ZONE_MCU_SHARED);
2262	3069	if (!va_reg)
2263	3070	goto va_region_alloc_error;
2264	3071
..	..	@@ -2272,9 +3079,9 @@
2272	3079	gpu_map_properties &= (KBASE_REG_GPU_RD \| KBASE_REG_GPU_WR);
2273	3080	gpu_map_properties \|= gpu_map_prot;
2274	3081
2275		- ret = kbase_mmu_insert_pages_no_flush(kbdev, &kbdev->csf.mcu_mmu,
2276		- va_reg->start_pfn, &phys[0], num_pages,
2277		- gpu_map_properties, KBASE_MEM_GROUP_CSF_FW);
	3082	+ ret = kbase_mmu_insert_pages_no_flush(kbdev, &kbdev->csf.mcu_mmu, va_reg->start_pfn,
	3083	+ &phys[0], num_pages, gpu_map_properties,
	3084	+ KBASE_MEM_GROUP_CSF_FW, NULL, NULL, false);
2278	3085	if (ret)
2279	3086	goto mmu_insert_pages_error;
2280	3087
..	..	@@ -2288,7 +3095,7 @@
2288	3095
2289	3096	mmu_insert_pages_error:
2290	3097	mutex_lock(&kbdev->csf.reg_lock);
2291		- kbase_remove_va_region(va_reg);
	3098	+ kbase_remove_va_region(kbdev, va_reg);
2292	3099	va_region_add_error:
2293	3100	kbase_free_alloced_region(va_reg);
2294	3101	mutex_unlock(&kbdev->csf.reg_lock);
..	..	@@ -2320,7 +3127,7 @@
2320	3127	{
2321	3128	if (csf_mapping->va_reg) {
2322	3129	mutex_lock(&kbdev->csf.reg_lock);
2323		- kbase_remove_va_region(csf_mapping->va_reg);
	3130	+ kbase_remove_va_region(kbdev, csf_mapping->va_reg);
2324	3131	kbase_free_alloced_region(csf_mapping->va_reg);
2325	3132	mutex_unlock(&kbdev->csf.reg_lock);
2326	3133	}