forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2024-01-04 1543e317f1da31b75942316931e8f491a8920811 
 kernel/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_csf.c
....@@ -1,7 +1,7 @@
11 // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
22 /*
33  *
4
- * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved.
4
+ * (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved.
55  *
66  * This program is free software and is provided to you under the terms of the
77  * GNU General Public License version 2 as published by the Free Software
....@@ -23,12 +23,13 @@
2323 #include <device/mali_kbase_device.h>
2424 
2525 #include <mali_kbase_hwaccess_backend.h>
26
-#include <mali_kbase_hwcnt_backend_csf_if_fw.h>
26
+#include <hwcnt/backend/mali_kbase_hwcnt_backend_csf_if_fw.h>
27
+#include <hwcnt/mali_kbase_hwcnt_watchdog_if_timer.h>
2728 #include <mali_kbase_ctx_sched.h>
2829 #include <mali_kbase_reset_gpu.h>
2930 #include <csf/mali_kbase_csf.h>
3031 #include <csf/ipa_control/mali_kbase_csf_ipa_control.h>
31
-
32
+#include <backend/gpu/mali_kbase_model_linux.h>
3233 
3334 #include <mali_kbase.h>
3435 #include <backend/gpu/mali_kbase_irq_internal.h>
....@@ -36,8 +37,10 @@
3637 #include <backend/gpu/mali_kbase_js_internal.h>
3738 #include <backend/gpu/mali_kbase_clk_rate_trace_mgr.h>
3839 #include <csf/mali_kbase_csf_csg_debugfs.h>
39
-#include <mali_kbase_hwcnt_virtualizer.h>
40
+#include <hwcnt/mali_kbase_hwcnt_virtualizer.h>
41
+#include <mali_kbase_kinstr_prfcnt.h>
4042 #include <mali_kbase_vinstr.h>
43
+#include <tl/mali_kbase_timeline.h>
4144 
4245 /**
4346  * kbase_device_firmware_hwcnt_term - Terminate CSF firmware and HWC
....@@ -51,10 +54,11 @@
5154 static void kbase_device_firmware_hwcnt_term(struct kbase_device *kbdev)
5255 {
5356 	if (kbdev->csf.firmware_inited) {
57
+		kbase_kinstr_prfcnt_term(kbdev->kinstr_prfcnt_ctx);
5458 		kbase_vinstr_term(kbdev->vinstr_ctx);
5559 		kbase_hwcnt_virtualizer_term(kbdev->hwcnt_gpu_virt);
5660 		kbase_hwcnt_backend_csf_metadata_term(&kbdev->hwcnt_gpu_iface);
57
-		kbase_csf_firmware_term(kbdev);
61
+		kbase_csf_firmware_unload_term(kbdev);
5862 	}
5963 }
6064 
....@@ -85,13 +89,13 @@
8589 		goto fail_timer;
8690 
8791 #ifdef CONFIG_MALI_BIFROST_DEBUG
88
-#ifndef CONFIG_MALI_BIFROST_NO_MALI
92
+#if IS_ENABLED(CONFIG_MALI_REAL_HW)
8993 	if (kbasep_common_test_interrupt_handlers(kbdev) != 0) {
9094 		dev_err(kbdev->dev, "Interrupt assignment check failed.\n");
9195 		err = -EINVAL;
9296 		goto fail_interrupt_test;
9397 	}
94
-#endif /* !CONFIG_MALI_BIFROST_NO_MALI */
98
+#endif /* IS_ENABLED(CONFIG_MALI_REAL_HW) */
9599 #endif /* CONFIG_MALI_BIFROST_DEBUG */
96100 
97101 	kbase_ipa_control_init(kbdev);
....@@ -119,6 +123,10 @@
119123 	if (err)
120124 		goto fail_update_l2_features;
121125 
126
+	err = kbase_backend_time_init(kbdev);
127
+	if (err)
128
+		goto fail_update_l2_features;
129
+
122130 	init_waitqueue_head(&kbdev->hwaccess.backend.reset_wait);
123131 
124132 	kbase_pm_context_idle(kbdev);
....@@ -135,9 +143,9 @@
135143 	kbase_ipa_control_term(kbdev);
136144 
137145 #ifdef CONFIG_MALI_BIFROST_DEBUG
138
-#ifndef CONFIG_MALI_BIFROST_NO_MALI
146
+#if IS_ENABLED(CONFIG_MALI_REAL_HW)
139147 fail_interrupt_test:
140
-#endif /* !CONFIG_MALI_BIFROST_NO_MALI */
148
+#endif /* IS_ENABLED(CONFIG_MALI_REAL_HW) */
141149 #endif /* CONFIG_MALI_BIFROST_DEBUG */
142150 
143151 	kbase_backend_timer_term(kbdev);
....@@ -185,18 +193,56 @@
185193 }
186194 
187195 /**
188
- * kbase_csf_early_init - Early termination for firmware & scheduler.
196
+ * kbase_csf_early_term() - Early termination for firmware & scheduler.
189197  * @kbdev:	Device pointer
190198  */
191199 static void kbase_csf_early_term(struct kbase_device *kbdev)
192200 {
193201 	kbase_csf_scheduler_early_term(kbdev);
202
+	kbase_csf_firmware_early_term(kbdev);
203
+}
204
+
205
+/**
206
+ * kbase_csf_late_init - late initialization for firmware.
207
+ * @kbdev:	Device pointer
208
+ *
209
+ * Return: 0 on success, error code otherwise.
210
+ */
211
+static int kbase_csf_late_init(struct kbase_device *kbdev)
212
+{
213
+	int err = kbase_csf_firmware_late_init(kbdev);
214
+
215
+	return err;
216
+}
217
+
218
+/**
219
+ * kbase_device_hwcnt_watchdog_if_init - Create hardware counter watchdog
220
+ *                                       interface.
221
+ * @kbdev:	Device pointer
222
+ *
223
+ * Return: 0 if successful or a negative error code on failure.
224
+ */
225
+static int kbase_device_hwcnt_watchdog_if_init(struct kbase_device *kbdev)
226
+{
227
+	return kbase_hwcnt_watchdog_if_timer_create(
228
+		&kbdev->hwcnt_watchdog_timer);
229
+}
230
+
231
+/**
232
+ * kbase_device_hwcnt_watchdog_if_term - Terminate hardware counter watchdog
233
+ *                                       interface.
234
+ * @kbdev:	Device pointer
235
+ */
236
+static void kbase_device_hwcnt_watchdog_if_term(struct kbase_device *kbdev)
237
+{
238
+	kbase_hwcnt_watchdog_if_timer_destroy(&kbdev->hwcnt_watchdog_timer);
194239 }
195240 
196241 /**
197242  * kbase_device_hwcnt_backend_csf_if_init - Create hardware counter backend
198243  *                                          firmware interface.
199244  * @kbdev:	Device pointer
245
+ * Return: 0 if successful or a negative error code on failure.
200246  */
201247 static int kbase_device_hwcnt_backend_csf_if_init(struct kbase_device *kbdev)
202248 {
....@@ -217,14 +263,15 @@
217263 /**
218264  * kbase_device_hwcnt_backend_csf_init - Create hardware counter backend.
219265  * @kbdev:	Device pointer
266
+ *
267
+ * Return: 0 if successful or a negative error code on failure.
220268  */
221
-
222269 static int kbase_device_hwcnt_backend_csf_init(struct kbase_device *kbdev)
223270 {
224271 	return kbase_hwcnt_backend_csf_create(
225272 		&kbdev->hwcnt_backend_csf_if_fw,
226273 		KBASE_HWCNT_BACKEND_CSF_RING_BUFFER_COUNT,
227
-		&kbdev->hwcnt_gpu_iface);
274
+		&kbdev->hwcnt_watchdog_timer, &kbdev->hwcnt_gpu_iface);
228275 }
229276 
230277 /**
....@@ -237,51 +284,52 @@
237284 }
238285 
239286 static const struct kbase_device_init dev_init[] = {
287
+#if !IS_ENABLED(CONFIG_MALI_REAL_HW)
288
+	{ kbase_gpu_device_create, kbase_gpu_device_destroy,
289
+	  "Dummy model initialization failed" },
290
+#else /* !IS_ENABLED(CONFIG_MALI_REAL_HW) */
240291 	{ assign_irqs, NULL, "IRQ search failed" },
292
+#endif /* !IS_ENABLED(CONFIG_MALI_REAL_HW) */
293
+#if !IS_ENABLED(CONFIG_MALI_BIFROST_NO_MALI)
241294 	{ registers_map, registers_unmap, "Register map failed" },
242
-	{ power_control_init, power_control_term,
243
-	  "Power control initialization failed" },
295
+#endif /* !IS_ENABLED(CONFIG_MALI_BIFROST_NO_MALI) */
296
+	{ power_control_init, power_control_term, "Power control initialization failed" },
244297 	{ kbase_device_io_history_init, kbase_device_io_history_term,
245298 	  "Register access history initialization failed" },
246
-	{ kbase_device_early_init, kbase_device_early_term,
247
-	  "Early device initialization failed" },
248
-	{ kbase_device_populate_max_freq, NULL,
249
-	  "Populating max frequency failed" },
299
+	{ kbase_device_early_init, kbase_device_early_term, "Early device initialization failed" },
300
+	{ kbase_device_populate_max_freq, NULL, "Populating max frequency failed" },
301
+	{ kbase_pm_lowest_gpu_freq_init, NULL, "Lowest freq initialization failed" },
250302 	{ kbase_device_misc_init, kbase_device_misc_term,
251303 	  "Miscellaneous device initialization failed" },
252304 	{ kbase_device_pcm_dev_init, kbase_device_pcm_dev_term,
253305 	  "Priority control manager initialization failed" },
254
-	{ kbase_ctx_sched_init, kbase_ctx_sched_term,
255
-	  "Context scheduler initialization failed" },
256
-	{ kbase_mem_init, kbase_mem_term,
257
-	  "Memory subsystem initialization failed" },
306
+	{ kbase_ctx_sched_init, kbase_ctx_sched_term, "Context scheduler initialization failed" },
307
+	{ kbase_mem_init, kbase_mem_term, "Memory subsystem initialization failed" },
258308 	{ kbase_csf_protected_memory_init, kbase_csf_protected_memory_term,
259309 	  "Protected memory allocator initialization failed" },
260310 	{ kbase_device_coherency_init, NULL, "Device coherency init failed" },
261311 	{ kbase_protected_mode_init, kbase_protected_mode_term,
262312 	  "Protected mode subsystem initialization failed" },
263
-	{ kbase_device_list_init, kbase_device_list_term,
264
-	  "Device list setup failed" },
313
+	{ kbase_device_list_init, kbase_device_list_term, "Device list setup failed" },
265314 	{ kbase_device_timeline_init, kbase_device_timeline_term,
266315 	  "Timeline stream initialization failed" },
267316 	{ kbase_clk_rate_trace_manager_init, kbase_clk_rate_trace_manager_term,
268317 	  "Clock rate trace manager initialization failed" },
269
-	{ kbase_device_hwcnt_backend_csf_if_init,
270
-	  kbase_device_hwcnt_backend_csf_if_term,
318
+	{ kbase_device_hwcnt_watchdog_if_init, kbase_device_hwcnt_watchdog_if_term,
319
+	  "GPU hwcnt backend watchdog interface creation failed" },
320
+	{ kbase_device_hwcnt_backend_csf_if_init, kbase_device_hwcnt_backend_csf_if_term,
271321 	  "GPU hwcnt backend CSF interface creation failed" },
272
-	{ kbase_device_hwcnt_backend_csf_init,
273
-	  kbase_device_hwcnt_backend_csf_term,
322
+	{ kbase_device_hwcnt_backend_csf_init, kbase_device_hwcnt_backend_csf_term,
274323 	  "GPU hwcnt backend creation failed" },
275324 	{ kbase_device_hwcnt_context_init, kbase_device_hwcnt_context_term,
276325 	  "GPU hwcnt context initialization failed" },
277
-	{ kbase_backend_late_init, kbase_backend_late_term,
278
-	  "Late backend initialization failed" },
279
-	{ kbase_csf_early_init, kbase_csf_early_term,
280
-	  "Early CSF initialization failed" },
326
+	{ kbase_csf_early_init, kbase_csf_early_term, "Early CSF initialization failed" },
327
+	{ kbase_backend_late_init, kbase_backend_late_term, "Late backend initialization failed" },
328
+	{ kbase_csf_late_init, NULL, "Late CSF initialization failed" },
281329 	{ NULL, kbase_device_firmware_hwcnt_term, NULL },
282
-#ifdef MALI_KBASE_BUILD
283
-	{ kbase_device_debugfs_init, kbase_device_debugfs_term,
284
-	  "DebugFS initialization failed" },
330
+	{ kbase_debug_csf_fault_init, kbase_debug_csf_fault_term,
331
+	  "CSF fault debug initialization failed" },
332
+	{ kbase_device_debugfs_init, kbase_device_debugfs_term, "DebugFS initialization failed" },
285333 	/* Sysfs init needs to happen before registering the device with
286334 	 * misc_register(), otherwise it causes a race condition between
287335 	 * registering the device and a uevent event being generated for
....@@ -299,9 +347,11 @@
299347 	  "Misc device registration failed" },
300348 	{ kbase_gpuprops_populate_user_buffer, kbase_gpuprops_free_user_buffer,
301349 	  "GPU property population failed" },
302
-	{ kbase_device_late_init, kbase_device_late_term,
303
-	  "Late device initialization failed" },
304
-#endif
350
+	{ kbase_device_late_init, kbase_device_late_term, "Late device initialization failed" },
351
+#if IS_ENABLED(CONFIG_MALI_CORESIGHT)
352
+	{ kbase_debug_coresight_csf_init, kbase_debug_coresight_csf_term,
353
+	  "Coresight initialization failed" },
354
+#endif /* IS_ENABLED(CONFIG_MALI_CORESIGHT) */
305355 };
306356 
307357 static void kbase_device_term_partial(struct kbase_device *kbdev,
....@@ -315,7 +365,6 @@
315365 
316366 void kbase_device_term(struct kbase_device *kbdev)
317367 {
318
-	kbdev->csf.mali_file_inode = NULL;
319368 	kbase_device_term_partial(kbdev, ARRAY_SIZE(dev_init));
320369 	kbase_mem_halt(kbdev);
321370 }
....@@ -354,7 +403,7 @@
354403  * Hardware counter components depending on firmware are initialized after CSF
355404  * firmware is loaded.
356405  *
357
- * @return 0 on success. An error code on failure.
406
+ * Return: 0 on success. An error code on failure.
358407  */
359408 static int kbase_device_hwcnt_csf_deferred_init(struct kbase_device *kbdev)
360409 {
....@@ -390,7 +439,18 @@
390439 		goto vinstr_fail;
391440 	}
392441 
442
+	ret = kbase_kinstr_prfcnt_init(kbdev->hwcnt_gpu_virt,
443
+				       &kbdev->kinstr_prfcnt_ctx);
444
+	if (ret) {
445
+		dev_err(kbdev->dev,
446
+			"Performance counter instrumentation initialization failed");
447
+		goto kinstr_prfcnt_fail;
448
+	}
449
+
393450 	return ret;
451
+
452
+kinstr_prfcnt_fail:
453
+	kbase_vinstr_term(kbdev->vinstr_ctx);
394454 
395455 vinstr_fail:
396456 	kbase_hwcnt_virtualizer_term(kbdev->hwcnt_gpu_virt);
....@@ -410,7 +470,7 @@
410470  * To meet Android GKI vendor guideline, firmware load is deferred at
411471  * the time when @ref kbase_open is called for the first time.
412472  *
413
- * @return 0 on success. An error code on failure.
473
+ * Return: 0 on success. An error code on failure.
414474  */
415475 static int kbase_csf_firmware_deferred_init(struct kbase_device *kbdev)
416476 {
....@@ -418,9 +478,7 @@
418478 
419479 	lockdep_assert_held(&kbdev->fw_load_lock);
420480 
421
-	kbase_pm_context_active(kbdev);
422
-
423
-	err = kbase_csf_firmware_init(kbdev);
481
+	err = kbase_csf_firmware_load_init(kbdev);
424482 	if (!err) {
425483 		unsigned long flags;
426484 
....@@ -432,8 +490,6 @@
432490 		dev_err(kbdev->dev, "Firmware initialization failed");
433491 	}
434492 
435
-	kbase_pm_context_idle(kbdev);
436
-
437493 	return err;
438494 }
439495 
....@@ -444,21 +500,26 @@
444500 	mutex_lock(&kbdev->fw_load_lock);
445501 
446502 	if (!kbdev->csf.firmware_inited) {
503
+		kbase_pm_context_active(kbdev);
504
+
447505 		ret = kbase_csf_firmware_deferred_init(kbdev);
448506 		if (ret)
449507 			goto out;
450508 
451509 		ret = kbase_device_hwcnt_csf_deferred_init(kbdev);
452510 		if (ret) {
453
-			kbase_csf_firmware_term(kbdev);
511
+			kbase_csf_firmware_unload_term(kbdev);
454512 			goto out;
455513 		}
456514 
457515 		kbase_csf_debugfs_init(kbdev);
516
+		kbase_timeline_io_debugfs_init(kbdev);
517
+out:
518
+		kbase_pm_context_idle(kbdev);
458519 	}
459520 
460
-out:
461521 	mutex_unlock(&kbdev->fw_load_lock);
462522 
463523 	return ret;
464524 }
525
+KBASE_EXPORT_TEST_API(kbase_device_firmware_init_once);