forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2024-01-03 2f7c68cb55ecb7331f2381deb497c27155f32faf 
 kernel/drivers/gpu/arm/bifrost/device/mali_kbase_device.c
....@@ -1,7 +1,7 @@
11 // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
22 /*
33  *
4
- * (C) COPYRIGHT 2010-2021 ARM Limited. All rights reserved.
4
+ * (C) COPYRIGHT 2010-2023 ARM Limited. All rights reserved.
55  *
66  * This program is free software and is provided to you under the terms of the
77  * GNU General Public License version 2 as published by the Free Software
....@@ -35,35 +35,36 @@
3535 #include <mali_kbase.h>
3636 #include <mali_kbase_defs.h>
3737 #include <mali_kbase_hwaccess_instr.h>
38
+#include <mali_kbase_hwaccess_time.h>
3839 #include <mali_kbase_hw.h>
3940 #include <mali_kbase_config_defaults.h>
4041 #include <linux/priority_control_manager.h>
4142 
4243 #include <tl/mali_kbase_timeline.h>
44
+#include "mali_kbase_kinstr_prfcnt.h"
4345 #include "mali_kbase_vinstr.h"
44
-#include "mali_kbase_hwcnt_context.h"
45
-#include "mali_kbase_hwcnt_virtualizer.h"
46
+#include "hwcnt/mali_kbase_hwcnt_context.h"
47
+#include "hwcnt/mali_kbase_hwcnt_virtualizer.h"
4648 
4749 #include "mali_kbase_device.h"
4850 #include "mali_kbase_device_internal.h"
4951 #include "backend/gpu/mali_kbase_pm_internal.h"
5052 #include "backend/gpu/mali_kbase_irq_internal.h"
5153 #include "mali_kbase_regs_history_debugfs.h"
54
+#include "mali_kbase_pbha.h"
5255 
5356 #ifdef CONFIG_MALI_ARBITER_SUPPORT
5457 #include "arbiter/mali_kbase_arbiter_pm.h"
5558 #endif /* CONFIG_MALI_ARBITER_SUPPORT */
5659 
57
-/* NOTE: Magic - 0x45435254 (TRCE in ASCII).
58
- * Supports tracing feature provided in the base module.
59
- * Please keep it in sync with the value of base module.
60
- */
61
-#define TRACE_BUFFER_HEADER_SPECIAL 0x45435254
60
+#if defined(CONFIG_DEBUG_FS) && !IS_ENABLED(CONFIG_MALI_BIFROST_NO_MALI)
6261 
6362 /* Number of register accesses for the buffer that we allocate during
6463  * initialization time. The buffer size can be changed later via debugfs.
6564  */
6665 #define KBASEP_DEFAULT_REGISTER_HISTORY_SIZE ((u16)512)
66
+
67
+#endif /* defined(CONFIG_DEBUG_FS) && !IS_ENABLED(CONFIG_MALI_BIFROST_NO_MALI) */
6768 
6869 static DEFINE_MUTEX(kbase_dev_list_lock);
6970 static LIST_HEAD(kbase_dev_list);
....@@ -161,11 +162,14 @@
161162 /**
162163  * mali_oom_notifier_handler - Mali driver out-of-memory handler
163164  *
164
- * @nb - notifier block - used to retrieve kbdev pointer
165
- * @action - action (unused)
166
- * @data - data pointer (unused)
165
+ * @nb: notifier block - used to retrieve kbdev pointer
166
+ * @action: action (unused)
167
+ * @data: data pointer (unused)
168
+ *
167169  * This function simply lists memory usage by the Mali driver, per GPU device,
168170  * for diagnostic purposes.
171
+ *
172
+ * Return: NOTIFY_OK on success, NOTIFY_BAD otherwise.
169173  */
170174 static int mali_oom_notifier_handler(struct notifier_block *nb,
171175 				     unsigned long action, void *data)
....@@ -187,19 +191,25 @@
187191 
188192 	mutex_lock(&kbdev->kctx_list_lock);
189193 
190
-	list_for_each_entry (kctx, &kbdev->kctx_list, kctx_list_link) {
194
+	list_for_each_entry(kctx, &kbdev->kctx_list, kctx_list_link) {
191195 		struct pid *pid_struct;
192196 		struct task_struct *task;
197
+		struct pid *tgid_struct;
198
+		struct task_struct *tgid_task;
199
+
193200 		unsigned long task_alloc_total =
194201 			KBASE_PAGES_TO_KIB(atomic_read(&(kctx->used_pages)));
195202 
196203 		rcu_read_lock();
197204 		pid_struct = find_get_pid(kctx->pid);
198205 		task = pid_task(pid_struct, PIDTYPE_PID);
206
+		tgid_struct = find_get_pid(kctx->tgid);
207
+		tgid_task = pid_task(tgid_struct, PIDTYPE_PID);
199208 
200209 		dev_err(kbdev->dev,
201
-			"OOM notifier: tsk %s  tgid (%u)  pid (%u) %lu kB\n",
202
-			task ? task->comm : "[null task]", kctx->tgid,
210
+			"OOM notifier: tsk %s:%s  tgid (%u)  pid (%u) %lu kB\n",
211
+			tgid_task ? tgid_task->comm : "[null task]",
212
+			task ? task->comm : "[null comm]", kctx->tgid,
203213 			kctx->pid, task_alloc_total);
204214 
205215 		put_pid(pid_struct);
....@@ -273,13 +283,20 @@
273283 	if (err)
274284 		goto dma_set_mask_failed;
275285 
286
+
287
+	/* There is no limit for Mali, so set to max. */
288
+	if (kbdev->dev->dma_parms)
289
+		err = dma_set_max_seg_size(kbdev->dev, UINT_MAX);
290
+	if (err)
291
+		goto dma_set_mask_failed;
292
+
276293 	kbdev->nr_hw_address_spaces = kbdev->gpu_props.num_address_spaces;
277294 
278295 	err = kbase_device_all_as_init(kbdev);
279296 	if (err)
280297 		goto dma_set_mask_failed;
281298 
282
-	err = kbase_ktrace_init(kbdev);
299
+	err = kbase_pbha_read_dtb(kbdev);
283300 	if (err)
284301 		goto term_as;
285302 
....@@ -291,10 +308,15 @@
291308 
292309 	kbdev->pm.dvfs_period = DEFAULT_PM_DVFS_PERIOD;
293310 
294
-	kbdev->reset_timeout_ms = DEFAULT_RESET_TIMEOUT_MS;
311
+#if MALI_USE_CSF
312
+	kbdev->reset_timeout_ms = kbase_get_timeout_ms(kbdev, CSF_CSG_SUSPEND_TIMEOUT);
313
+#else
314
+	kbdev->reset_timeout_ms = JM_DEFAULT_RESET_TIMEOUT_MS;
315
+#endif /* MALI_USE_CSF */
295316 
296317 	kbdev->mmu_mode = kbase_mmu_mode_get_aarch64();
297
-
318
+	kbdev->mmu_as_inactive_wait_time_ms =
319
+		kbase_get_timeout_ms(kbdev, MMU_AS_INACTIVE_WAIT_TIMEOUT);
298320 	mutex_init(&kbdev->kctx_list_lock);
299321 	INIT_LIST_HEAD(&kbdev->kctx_list);
300322 
....@@ -307,6 +329,16 @@
307329 			"Unable to register OOM notifier for Mali - but will continue\n");
308330 		kbdev->oom_notifier_block.notifier_call = NULL;
309331 	}
332
+
333
+#if !MALI_USE_CSF
334
+	spin_lock_init(&kbdev->quick_reset_lock);
335
+	kbdev->quick_reset_enabled = true;
336
+	kbdev->num_of_atoms_hw_completed = 0;
337
+#endif
338
+
339
+#if MALI_USE_CSF && IS_ENABLED(CONFIG_SYNC_FILE)
340
+	atomic_set(&kbdev->live_fence_metadata, 0);
341
+#endif
310342 	return 0;
311343 
312344 term_as:
....@@ -325,14 +357,44 @@
325357 #if KBASE_KTRACE_ENABLE
326358 	kbase_debug_assert_register_hook(NULL, NULL);
327359 #endif
328
-
329
-	kbase_ktrace_term(kbdev);
330
-
331360 	kbase_device_all_as_term(kbdev);
361
+
332362 
333363 	if (kbdev->oom_notifier_block.notifier_call)
334364 		unregister_oom_notifier(&kbdev->oom_notifier_block);
365
+
366
+#if MALI_USE_CSF && IS_ENABLED(CONFIG_SYNC_FILE)
367
+	if (atomic_read(&kbdev->live_fence_metadata) > 0)
368
+		dev_warn(kbdev->dev, "Terminating Kbase device with live fence metadata!");
369
+#endif
335370 }
371
+
372
+#if !MALI_USE_CSF
373
+void kbase_enable_quick_reset(struct kbase_device *kbdev)
374
+{
375
+	spin_lock(&kbdev->quick_reset_lock);
376
+
377
+	kbdev->quick_reset_enabled = true;
378
+	kbdev->num_of_atoms_hw_completed = 0;
379
+
380
+	spin_unlock(&kbdev->quick_reset_lock);
381
+}
382
+
383
+void kbase_disable_quick_reset(struct kbase_device *kbdev)
384
+{
385
+	spin_lock(&kbdev->quick_reset_lock);
386
+
387
+	kbdev->quick_reset_enabled = false;
388
+	kbdev->num_of_atoms_hw_completed = 0;
389
+
390
+	spin_unlock(&kbdev->quick_reset_lock);
391
+}
392
+
393
+bool kbase_is_quick_reset_enabled(struct kbase_device *kbdev)
394
+{
395
+	return kbdev->quick_reset_enabled;
396
+}
397
+#endif
336398 
337399 void kbase_device_free(struct kbase_device *kbdev)
338400 {
....@@ -395,7 +457,17 @@
395457 	kbase_vinstr_term(kbdev->vinstr_ctx);
396458 }
397459 
398
-#if defined(CONFIG_DEBUG_FS) && !IS_ENABLED(CONFIG_MALI_BIFROST_NO_MALI)
460
+int kbase_device_kinstr_prfcnt_init(struct kbase_device *kbdev)
461
+{
462
+	return kbase_kinstr_prfcnt_init(kbdev->hwcnt_gpu_virt,
463
+					&kbdev->kinstr_prfcnt_ctx);
464
+}
465
+
466
+void kbase_device_kinstr_prfcnt_term(struct kbase_device *kbdev)
467
+{
468
+	kbase_kinstr_prfcnt_term(kbdev->kinstr_prfcnt_ctx);
469
+}
470
+
399471 int kbase_device_io_history_init(struct kbase_device *kbdev)
400472 {
401473 	return kbase_io_history_init(&kbdev->io_history,
....@@ -406,7 +478,6 @@
406478 {
407479 	kbase_io_history_term(&kbdev->io_history);
408480 }
409
-#endif
410481 
411482 int kbase_device_misc_register(struct kbase_device *kbdev)
412483 {
....@@ -455,27 +526,38 @@
455526 {
456527 	int err;
457528 
458
-	err = kbasep_platform_device_init(kbdev);
529
+	err = kbase_ktrace_init(kbdev);
459530 	if (err)
460531 		return err;
532
+
533
+
534
+	err = kbasep_platform_device_init(kbdev);
535
+	if (err)
536
+		goto ktrace_term;
461537 
462538 	err = kbase_pm_runtime_init(kbdev);
463539 	if (err)
464540 		goto fail_runtime_pm;
465541 
542
+	/* This spinlock is initialized before doing the first access to GPU
543
+	 * registers and installing interrupt handlers.
544
+	 */
545
+	spin_lock_init(&kbdev->hwaccess_lock);
546
+
466547 	/* Ensure we can access the GPU registers */
467548 	kbase_pm_register_access_enable(kbdev);
468549 
469
-	/* Find out GPU properties based on the GPU feature registers */
550
+	/*
551
+	 * Find out GPU properties based on the GPU feature registers.
552
+	 * Note that this does not populate the few properties that depend on
553
+	 * hw_features being initialized. Those are set by kbase_gpuprops_set_features
554
+	 * soon after this in the init process.
555
+	 */
470556 	kbase_gpuprops_set(kbdev);
471557 
472558 	/* We're done accessing the GPU registers for now. */
473559 	kbase_pm_register_access_disable(kbdev);
474560 
475
-	/* This spinlock has to be initialized before installing interrupt
476
-	 * handlers that require to hold it to process interrupts.
477
-	 */
478
-	spin_lock_init(&kbdev->hwaccess_lock);
479561 #ifdef CONFIG_MALI_ARBITER_SUPPORT
480562 	if (kbdev->arb.arb_if)
481563 		err = kbase_arbiter_pm_install_interrupts(kbdev);
....@@ -493,6 +575,8 @@
493575 	kbase_pm_runtime_term(kbdev);
494576 fail_runtime_pm:
495577 	kbasep_platform_device_term(kbdev);
578
+ktrace_term:
579
+	kbase_ktrace_term(kbdev);
496580 
497581 	return err;
498582 }
....@@ -509,6 +593,7 @@
509593 #endif /* CONFIG_MALI_ARBITER_SUPPORT */
510594 	kbase_pm_runtime_term(kbdev);
511595 	kbasep_platform_device_term(kbdev);
596
+	kbase_ktrace_term(kbdev);
512597 }
513598 
514599 int kbase_device_late_init(struct kbase_device *kbdev)