hc
2024-01-03 2f7c68cb55ecb7331f2381deb497c27155f32faf
kernel/drivers/gpu/arm/bifrost/csf/ipa_control/mali_kbase_csf_ipa_control.c
....@@ -1,7 +1,7 @@
11 // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
22 /*
33 *
4
- * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved.
4
+ * (C) COPYRIGHT 2020-2022 ARM Limited. All rights reserved.
55 *
66 * This program is free software and is provided to you under the terms of the
77 * GNU General Public License version 2 as published by the Free Software
....@@ -20,6 +20,7 @@
2020 */
2121
2222 #include <mali_kbase.h>
23
+#include <mali_kbase_config_defaults.h>
2324 #include "backend/gpu/mali_kbase_clk_rate_trace_mgr.h"
2425 #include "mali_kbase_csf_ipa_control.h"
2526
....@@ -27,8 +28,6 @@
2728 * Status flags from the STATUS register of the IPA Control interface.
2829 */
2930 #define STATUS_COMMAND_ACTIVE ((u32)1 << 0)
30
-#define STATUS_TIMER_ACTIVE ((u32)1 << 1)
31
-#define STATUS_AUTO_ACTIVE ((u32)1 << 2)
3231 #define STATUS_PROTECTED_MODE ((u32)1 << 8)
3332 #define STATUS_RESET ((u32)1 << 9)
3433 #define STATUS_TIMER_ENABLED ((u32)1 << 31)
....@@ -36,39 +35,27 @@
3635 /*
3736 * Commands for the COMMAND register of the IPA Control interface.
3837 */
39
-#define COMMAND_NOP ((u32)0)
4038 #define COMMAND_APPLY ((u32)1)
41
-#define COMMAND_CLEAR ((u32)2)
4239 #define COMMAND_SAMPLE ((u32)3)
4340 #define COMMAND_PROTECTED_ACK ((u32)4)
4441 #define COMMAND_RESET_ACK ((u32)5)
4542
46
-/**
47
- * Default value for the TIMER register of the IPA Control interface,
48
- * expressed in milliseconds.
49
- *
50
- * The chosen value is a trade off between two requirements: the IPA Control
51
- * interface should sample counters with a resolution in the order of
52
- * milliseconds, while keeping GPU overhead as limited as possible.
53
- */
54
-#define TIMER_DEFAULT_VALUE_MS ((u32)10) /* 10 milliseconds */
55
-
56
-/**
43
+/*
5744 * Number of timer events per second.
5845 */
59
-#define TIMER_EVENTS_PER_SECOND ((u32)1000 / TIMER_DEFAULT_VALUE_MS)
46
+#define TIMER_EVENTS_PER_SECOND ((u32)1000 / IPA_CONTROL_TIMER_DEFAULT_VALUE_MS)
6047
61
-/**
48
+/*
6249 * Maximum number of loops polling the GPU before we assume the GPU has hung.
6350 */
64
-#define IPA_INACTIVE_MAX_LOOPS ((unsigned int)8000000)
51
+#define IPA_INACTIVE_MAX_LOOPS (8000000U)
6552
66
-/**
53
+/*
6754 * Number of bits used to configure a performance counter in SELECT registers.
6855 */
6956 #define IPA_CONTROL_SELECT_BITS_PER_CNT ((u64)8)
7057
71
-/**
58
+/*
7259 * Maximum value of a performance counter.
7360 */
7461 #define MAX_PRFCNT_VALUE (((u64)1 << 48) - 1)
....@@ -146,8 +133,12 @@
146133
147134 ret = wait_status(kbdev, STATUS_COMMAND_ACTIVE);
148135
149
- if (!ret)
136
+ if (!ret) {
150137 kbase_reg_write(kbdev, IPA_CONTROL_REG(COMMAND), COMMAND_APPLY);
138
+ ret = wait_status(kbdev, STATUS_COMMAND_ACTIVE);
139
+ } else {
140
+ dev_err(kbdev->dev, "Wait for the pending command failed");
141
+ }
151142
152143 return ret;
153144 }
....@@ -215,6 +206,17 @@
215206 }
216207 }
217208
209
+static int update_select_registers(struct kbase_device *kbdev)
210
+{
211
+ u64 select_config[KBASE_IPA_CORE_TYPE_NUM];
212
+
213
+ lockdep_assert_held(&kbdev->csf.ipa_control.lock);
214
+
215
+ build_select_config(&kbdev->csf.ipa_control, select_config);
216
+
217
+ return apply_select_config(kbdev, select_config);
218
+}
219
+
218220 static inline void calc_prfcnt_delta(struct kbase_device *kbdev,
219221 struct kbase_ipa_control_prfcnt *prfcnt,
220222 bool gpu_ready)
....@@ -236,9 +238,15 @@
236238
237239 delta_value *= prfcnt->scaling_factor;
238240
239
- if (!WARN_ON_ONCE(kbdev->csf.ipa_control.cur_gpu_rate == 0))
240
- if (prfcnt->gpu_norm)
241
- delta_value /= kbdev->csf.ipa_control.cur_gpu_rate;
241
+ if (kbdev->csf.ipa_control.cur_gpu_rate == 0) {
242
+ static bool warned;
243
+
244
+ if (!warned) {
245
+ dev_warn(kbdev->dev, "%s: GPU freq is unexpectedly 0", __func__);
246
+ warned = true;
247
+ }
248
+ } else if (prfcnt->gpu_norm)
249
+ delta_value = div_u64(delta_value, kbdev->csf.ipa_control.cur_gpu_rate);
242250
243251 prfcnt->latest_raw_value = raw_value;
244252
....@@ -285,17 +293,20 @@
285293 /* Interrupts are already disabled and interrupt state is also saved */
286294 spin_lock(&ipa_ctrl->lock);
287295
288
- for (i = 0; i < ipa_ctrl->num_active_sessions; i++) {
289
- size_t j;
296
+ for (i = 0; i < KBASE_IPA_CONTROL_MAX_SESSIONS; i++) {
290297 struct kbase_ipa_control_session *session = &ipa_ctrl->sessions[i];
291298
292
- for (j = 0; j < session->num_prfcnts; j++) {
293
- struct kbase_ipa_control_prfcnt *prfcnt =
294
- &session->prfcnts[j];
299
+ if (session->active) {
300
+ size_t j;
295301
296
- if (prfcnt->gpu_norm)
297
- calc_prfcnt_delta(kbdev, prfcnt, true);
298
- }
302
+ for (j = 0; j < session->num_prfcnts; j++) {
303
+ struct kbase_ipa_control_prfcnt *prfcnt =
304
+ &session->prfcnts[j];
305
+
306
+ if (prfcnt->gpu_norm)
307
+ calc_prfcnt_delta(kbdev, prfcnt, true);
308
+ }
309
+ }
299310 }
300311
301312 ipa_ctrl->cur_gpu_rate = clk_rate_hz;
....@@ -332,9 +343,8 @@
332343
333344 spin_lock_init(&ipa_ctrl->lock);
334345 ipa_ctrl->num_active_sessions = 0;
335
- for (i = 0; i < KBASE_IPA_CONTROL_MAX_SESSIONS; i++) {
346
+ for (i = 0; i < KBASE_IPA_CONTROL_MAX_SESSIONS; i++)
336347 ipa_ctrl->sessions[i].active = false;
337
- }
338348
339349 listener_data = kmalloc(sizeof(struct kbase_ipa_control_listener_data),
340350 GFP_KERNEL);
....@@ -377,6 +387,115 @@
377387 }
378388 KBASE_EXPORT_TEST_API(kbase_ipa_control_term);
379389
390
+/** session_read_raw_values - Read latest raw values for a sessions
391
+ * @kbdev: Pointer to kbase device.
392
+ * @session: Pointer to the session whose performance counters shall be read.
393
+ *
394
+ * Read and update the latest raw values of all the performance counters
395
+ * belonging to a given session.
396
+ */
397
+static void session_read_raw_values(struct kbase_device *kbdev,
398
+ struct kbase_ipa_control_session *session)
399
+{
400
+ size_t i;
401
+
402
+ lockdep_assert_held(&kbdev->csf.ipa_control.lock);
403
+
404
+ for (i = 0; i < session->num_prfcnts; i++) {
405
+ struct kbase_ipa_control_prfcnt *prfcnt = &session->prfcnts[i];
406
+ u64 raw_value = read_value_cnt(kbdev, (u8)prfcnt->type,
407
+ prfcnt->select_idx);
408
+
409
+ prfcnt->latest_raw_value = raw_value;
410
+ }
411
+}
412
+
413
+/** session_gpu_start - Start one or all sessions
414
+ * @kbdev: Pointer to kbase device.
415
+ * @ipa_ctrl: Pointer to IPA_CONTROL descriptor.
416
+ * @session: Pointer to the session to initialize, or NULL to initialize
417
+ * all sessions.
418
+ *
419
+ * This function starts one or all sessions by capturing a manual sample,
420
+ * reading the latest raw value of performance counters and possibly enabling
421
+ * the timer for automatic sampling if necessary.
422
+ *
423
+ * If a single session is given, it is assumed to be active, regardless of
424
+ * the number of active sessions. The number of performance counters belonging
425
+ * to the session shall be set in advance.
426
+ *
427
+ * If no session is given, the function shall start all sessions.
428
+ * The function does nothing if there are no active sessions.
429
+ *
430
+ * Return: 0 on success, or error code on failure.
431
+ */
432
+static int session_gpu_start(struct kbase_device *kbdev,
433
+ struct kbase_ipa_control *ipa_ctrl,
434
+ struct kbase_ipa_control_session *session)
435
+{
436
+ bool first_start =
437
+ (session != NULL) && (ipa_ctrl->num_active_sessions == 0);
438
+ int ret = 0;
439
+
440
+ lockdep_assert_held(&kbdev->csf.ipa_control.lock);
441
+
442
+ /*
443
+ * Exit immediately if the caller intends to start all sessions
444
+ * but there are no active sessions. It's important that no operation
445
+ * is done on the IPA_CONTROL interface in that case.
446
+ */
447
+ if (!session && ipa_ctrl->num_active_sessions == 0)
448
+ return ret;
449
+
450
+ /*
451
+ * Take a manual sample unconditionally if the caller intends
452
+ * to start all sessions. Otherwise, only take a manual sample
453
+ * if this is the first session to be initialized, for accumulator
454
+ * registers are empty and no timer has been configured for automatic
455
+ * sampling.
456
+ */
457
+ if (!session || first_start) {
458
+ kbase_reg_write(kbdev, IPA_CONTROL_REG(COMMAND),
459
+ COMMAND_SAMPLE);
460
+ ret = wait_status(kbdev, STATUS_COMMAND_ACTIVE);
461
+ if (ret)
462
+ dev_err(kbdev->dev, "%s: failed to sample new counters",
463
+ __func__);
464
+ kbase_reg_write(kbdev, IPA_CONTROL_REG(TIMER),
465
+ timer_value(ipa_ctrl->cur_gpu_rate));
466
+ }
467
+
468
+ /*
469
+ * Read current raw value to start the session.
470
+ * This is necessary to put the first query in condition
471
+ * to generate a correct value by calculating the difference
472
+ * from the beginning of the session. This consideration
473
+ * is true regardless of the number of sessions the caller
474
+ * intends to start.
475
+ */
476
+ if (!ret) {
477
+ if (session) {
478
+ /* On starting a session, value read is required for
479
+ * IPA power model's calculation initialization.
480
+ */
481
+ session_read_raw_values(kbdev, session);
482
+ } else {
483
+ size_t session_idx;
484
+
485
+ for (session_idx = 0;
486
+ session_idx < KBASE_IPA_CONTROL_MAX_SESSIONS;
487
+ session_idx++) {
488
+ struct kbase_ipa_control_session *session_to_check = &ipa_ctrl->sessions[session_idx];
489
+
490
+ if (session_to_check->active)
491
+ session_read_raw_values(kbdev, session_to_check);
492
+ }
493
+ }
494
+ }
495
+
496
+ return ret;
497
+}
498
+
380499 int kbase_ipa_control_register(
381500 struct kbase_device *kbdev,
382501 const struct kbase_ipa_control_perf_counter *perf_counters,
....@@ -390,8 +509,10 @@
390509 struct kbase_ipa_control_session *session = NULL;
391510 unsigned long flags;
392511
393
- if (WARN_ON(kbdev == NULL) || WARN_ON(perf_counters == NULL) ||
394
- WARN_ON(client == NULL) ||
512
+ if (WARN_ON(unlikely(kbdev == NULL)))
513
+ return -ENODEV;
514
+
515
+ if (WARN_ON(perf_counters == NULL) || WARN_ON(client == NULL) ||
395516 WARN_ON(num_counters > KBASE_IPA_CONTROL_MAX_COUNTERS)) {
396517 dev_err(kbdev->dev, "%s: wrong input arguments", __func__);
397518 return -EINVAL;
....@@ -468,9 +589,10 @@
468589 */
469590 for (session_idx = 0; session_idx < KBASE_IPA_CONTROL_MAX_SESSIONS;
470591 session_idx++) {
471
- session = &ipa_ctrl->sessions[session_idx];
472
- if (!session->active)
592
+ if (!ipa_ctrl->sessions[session_idx].active) {
593
+ session = &ipa_ctrl->sessions[session_idx];
473594 break;
595
+ }
474596 }
475597
476598 if (!session) {
....@@ -525,7 +647,7 @@
525647 /* Reports to this client for GPU time spent in protected mode
526648 * should begin from the point of registration.
527649 */
528
- session->last_query_time = ktime_get_ns();
650
+ session->last_query_time = ktime_get_raw_ns();
529651
530652 /* Initially, no time has been spent in protected mode */
531653 session->protm_time = 0;
....@@ -539,56 +661,22 @@
539661 * before applying the new configuration.
540662 */
541663 if (new_config) {
542
- u64 select_config[KBASE_IPA_CORE_TYPE_NUM];
543
-
544
- build_select_config(ipa_ctrl, select_config);
545
- ret = apply_select_config(kbdev, select_config);
664
+ ret = update_select_registers(kbdev);
546665 if (ret)
547666 dev_err(kbdev->dev,
548
- "%s: failed to apply SELECT configuration",
667
+ "%s: failed to apply new SELECT configuration",
549668 __func__);
550669 }
551670
552671 if (!ret) {
553
- /* Accumulator registers don't contain any sample if the timer
554
- * has not been enabled first. Take a sample manually before
555
- * enabling the timer.
556
- */
557
- if (ipa_ctrl->num_active_sessions == 0) {
558
- kbase_reg_write(kbdev, IPA_CONTROL_REG(COMMAND),
559
- COMMAND_SAMPLE);
560
- ret = wait_status(kbdev, STATUS_COMMAND_ACTIVE);
561
- if (!ret) {
562
- kbase_reg_write(
563
- kbdev, IPA_CONTROL_REG(TIMER),
564
- timer_value(ipa_ctrl->cur_gpu_rate));
565
- } else {
566
- dev_err(kbdev->dev,
567
- "%s: failed to sample new counters",
568
- __func__);
569
- }
570
- }
672
+ session->num_prfcnts = num_counters;
673
+ ret = session_gpu_start(kbdev, ipa_ctrl, session);
571674 }
572675
573676 if (!ret) {
574
- session->num_prfcnts = num_counters;
575677 session->active = true;
576678 ipa_ctrl->num_active_sessions++;
577679 *client = session;
578
-
579
- /*
580
- * Read current raw value to initialize the session.
581
- * This is necessary to put the first query in condition
582
- * to generate a correct value by calculating the difference
583
- * from the beginning of the session.
584
- */
585
- for (i = 0; i < session->num_prfcnts; i++) {
586
- struct kbase_ipa_control_prfcnt *prfcnt =
587
- &session->prfcnts[i];
588
- u64 raw_value = read_value_cnt(kbdev, (u8)prfcnt->type,
589
- prfcnt->select_idx);
590
- prfcnt->latest_raw_value = raw_value;
591
- }
592680 }
593681
594682 exit:
....@@ -607,7 +695,10 @@
607695 unsigned long flags;
608696 bool new_config = false, valid_session = false;
609697
610
- if (WARN_ON(kbdev == NULL) || WARN_ON(client == NULL)) {
698
+ if (WARN_ON(unlikely(kbdev == NULL)))
699
+ return -ENODEV;
700
+
701
+ if (WARN_ON(client == NULL)) {
611702 dev_err(kbdev->dev, "%s: wrong input arguments", __func__);
612703 return -EINVAL;
613704 }
....@@ -662,10 +753,7 @@
662753 }
663754
664755 if (new_config) {
665
- u64 select_config[KBASE_IPA_CORE_TYPE_NUM];
666
-
667
- build_select_config(ipa_ctrl, select_config);
668
- ret = apply_select_config(kbdev, select_config);
756
+ ret = update_select_registers(kbdev);
669757 if (ret)
670758 dev_err(kbdev->dev,
671759 "%s: failed to apply SELECT configuration",
....@@ -692,14 +780,22 @@
692780 unsigned long flags;
693781 bool gpu_ready;
694782
695
- if (WARN_ON(kbdev == NULL) || WARN_ON(client == NULL) ||
696
- WARN_ON(values == NULL)) {
783
+ if (WARN_ON(unlikely(kbdev == NULL)))
784
+ return -ENODEV;
785
+
786
+ if (WARN_ON(client == NULL) || WARN_ON(values == NULL)) {
697787 dev_err(kbdev->dev, "%s: wrong input arguments", __func__);
698788 return -EINVAL;
699789 }
700790
701791 ipa_ctrl = &kbdev->csf.ipa_control;
702792 session = (struct kbase_ipa_control_session *)client;
793
+
794
+ if (!session->active) {
795
+ dev_err(kbdev->dev,
796
+ "%s: attempt to query inactive session", __func__);
797
+ return -EINVAL;
798
+ }
703799
704800 if (WARN_ON(num_values < session->num_prfcnts)) {
705801 dev_err(kbdev->dev,
....@@ -721,7 +817,7 @@
721817 }
722818
723819 if (protected_time) {
724
- u64 time_now = ktime_get_ns();
820
+ u64 time_now = ktime_get_raw_ns();
725821
726822 /* This is the amount of protected-mode time spent prior to
727823 * the current protm period.
....@@ -778,20 +874,23 @@
778874 ret);
779875 }
780876
781
- for (session_idx = 0; session_idx < ipa_ctrl->num_active_sessions;
877
+ for (session_idx = 0; session_idx < KBASE_IPA_CONTROL_MAX_SESSIONS;
782878 session_idx++) {
879
+
783880 struct kbase_ipa_control_session *session =
784881 &ipa_ctrl->sessions[session_idx];
785
- size_t i;
786882
787
- for (i = 0; i < session->num_prfcnts; i++) {
788
- struct kbase_ipa_control_prfcnt *prfcnt =
789
- &session->prfcnts[i];
883
+ if (session->active) {
884
+ size_t i;
790885
791
- calc_prfcnt_delta(kbdev, prfcnt, true);
886
+ for (i = 0; i < session->num_prfcnts; i++) {
887
+ struct kbase_ipa_control_prfcnt *prfcnt =
888
+ &session->prfcnts[i];
889
+
890
+ calc_prfcnt_delta(kbdev, prfcnt, true);
891
+ }
792892 }
793893 }
794
-
795894 spin_unlock(&ipa_ctrl->lock);
796895 }
797896
....@@ -808,18 +907,16 @@
808907 /* Interrupts are already disabled and interrupt state is also saved */
809908 spin_lock(&ipa_ctrl->lock);
810909
811
- /* Re-issue the APPLY command, this is actually needed only for CSHW */
812
- kbase_reg_write(kbdev, IPA_CONTROL_REG(COMMAND), COMMAND_APPLY);
813
- ret = wait_status(kbdev, STATUS_COMMAND_ACTIVE);
910
+ ret = update_select_registers(kbdev);
814911 if (ret) {
815912 dev_err(kbdev->dev,
816
- "Wait for the completion of apply command failed: %d",
817
- ret);
913
+ "Failed to reconfigure the select registers: %d", ret);
818914 }
819915
820
- /* Re-enable the timer for periodic sampling */
821
- kbase_reg_write(kbdev, IPA_CONTROL_REG(TIMER),
822
- timer_value(ipa_ctrl->cur_gpu_rate));
916
+ /* Accumulator registers would not contain any sample after GPU power
917
+ * cycle if the timer has not been enabled first. Initialize all sessions.
918
+ */
919
+ ret = session_gpu_start(kbdev, ipa_ctrl, NULL);
823920
824921 spin_unlock(&ipa_ctrl->lock);
825922 }
....@@ -864,6 +961,43 @@
864961 }
865962 KBASE_EXPORT_TEST_API(kbase_ipa_control_handle_gpu_reset_post);
866963
964
+#ifdef KBASE_PM_RUNTIME
965
+void kbase_ipa_control_handle_gpu_sleep_enter(struct kbase_device *kbdev)
966
+{
967
+ lockdep_assert_held(&kbdev->hwaccess_lock);
968
+
969
+ if (kbdev->pm.backend.mcu_state == KBASE_MCU_IN_SLEEP) {
970
+ /* GPU Sleep is treated as a power down */
971
+ kbase_ipa_control_handle_gpu_power_off(kbdev);
972
+
973
+ /* SELECT_CSHW register needs to be cleared to prevent any
974
+ * IPA control message to be sent to the top level GPU HWCNT.
975
+ */
976
+ kbase_reg_write(kbdev, IPA_CONTROL_REG(SELECT_CSHW_LO), 0);
977
+ kbase_reg_write(kbdev, IPA_CONTROL_REG(SELECT_CSHW_HI), 0);
978
+
979
+ /* No need to issue the APPLY command here */
980
+ }
981
+}
982
+KBASE_EXPORT_TEST_API(kbase_ipa_control_handle_gpu_sleep_enter);
983
+
984
+void kbase_ipa_control_handle_gpu_sleep_exit(struct kbase_device *kbdev)
985
+{
986
+ lockdep_assert_held(&kbdev->hwaccess_lock);
987
+
988
+ if (kbdev->pm.backend.mcu_state == KBASE_MCU_IN_SLEEP) {
989
+ /* To keep things simple, currently exit from
990
+ * GPU Sleep is treated as a power on event where
991
+ * all 4 SELECT registers are reconfigured.
992
+ * On exit from sleep, reconfiguration is needed
993
+ * only for the SELECT_CSHW register.
994
+ */
995
+ kbase_ipa_control_handle_gpu_power_on(kbdev);
996
+ }
997
+}
998
+KBASE_EXPORT_TEST_API(kbase_ipa_control_handle_gpu_sleep_exit);
999
+#endif
1000
+
8671001 #if MALI_UNIT_TEST
8681002 void kbase_ipa_control_rate_change_notify_test(struct kbase_device *kbdev,
8691003 u32 clk_index, u32 clk_rate_hz)
....@@ -883,25 +1017,29 @@
8831017 struct kbase_ipa_control *ipa_ctrl = &kbdev->csf.ipa_control;
8841018
8851019 lockdep_assert_held(&kbdev->hwaccess_lock);
886
- ipa_ctrl->protm_start = ktime_get_ns();
1020
+ ipa_ctrl->protm_start = ktime_get_raw_ns();
8871021 }
8881022
8891023 void kbase_ipa_control_protm_exited(struct kbase_device *kbdev)
8901024 {
8911025 struct kbase_ipa_control *ipa_ctrl = &kbdev->csf.ipa_control;
8921026 size_t i;
893
- u64 time_now = ktime_get_ns();
1027
+ u64 time_now = ktime_get_raw_ns();
8941028 u32 status;
8951029
8961030 lockdep_assert_held(&kbdev->hwaccess_lock);
8971031
898
- for (i = 0; i < ipa_ctrl->num_active_sessions; i++) {
1032
+ for (i = 0; i < KBASE_IPA_CONTROL_MAX_SESSIONS; i++) {
1033
+
8991034 struct kbase_ipa_control_session *session =
9001035 &ipa_ctrl->sessions[i];
901
- u64 protm_time = time_now - MAX(session->last_query_time,
902
- ipa_ctrl->protm_start);
9031036
904
- session->protm_time += protm_time;
1037
+ if (session->active) {
1038
+ u64 protm_time = time_now - MAX(session->last_query_time,
1039
+ ipa_ctrl->protm_start);
1040
+
1041
+ session->protm_time += protm_time;
1042
+ }
9051043 }
9061044
9071045 /* Acknowledge the protected_mode bit in the IPA_CONTROL STATUS