forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2023-12-06 08f87f769b595151be1afeff53e144f543faa614 
 kernel/drivers/gpu/arm/bifrost/device/mali_kbase_device_hw.c
....@@ -1,7 +1,7 @@
11 // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
22 /*
33  *
4
- * (C) COPYRIGHT 2014-2016, 2018-2021 ARM Limited. All rights reserved.
4
+ * (C) COPYRIGHT 2014-2016, 2018-2022 ARM Limited. All rights reserved.
55  *
66  * This program is free software and is provided to you under the terms of the
77  * GNU General Public License version 2 as published by the Free Software
....@@ -28,44 +28,6 @@
2828 #include <mmu/mali_kbase_mmu.h>
2929 
3030 #if !IS_ENABLED(CONFIG_MALI_BIFROST_NO_MALI)
31
-void kbase_reg_write(struct kbase_device *kbdev, u32 offset, u32 value)
32
-{
33
-	KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_powered);
34
-	KBASE_DEBUG_ASSERT(kbdev->dev != NULL);
35
-
36
-	writel(value, kbdev->reg + offset);
37
-
38
-#if IS_ENABLED(CONFIG_DEBUG_FS)
39
-	if (unlikely(kbdev->io_history.enabled))
40
-		kbase_io_history_add(&kbdev->io_history, kbdev->reg + offset,
41
-				value, 1);
42
-#endif /* CONFIG_DEBUG_FS */
43
-	dev_dbg(kbdev->dev, "w: reg %08x val %08x", offset, value);
44
-}
45
-
46
-KBASE_EXPORT_TEST_API(kbase_reg_write);
47
-
48
-u32 kbase_reg_read(struct kbase_device *kbdev, u32 offset)
49
-{
50
-	u32 val;
51
-
52
-	KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_powered);
53
-	KBASE_DEBUG_ASSERT(kbdev->dev != NULL);
54
-
55
-	val = readl(kbdev->reg + offset);
56
-
57
-#if IS_ENABLED(CONFIG_DEBUG_FS)
58
-	if (unlikely(kbdev->io_history.enabled))
59
-		kbase_io_history_add(&kbdev->io_history, kbdev->reg + offset,
60
-				val, 0);
61
-#endif /* CONFIG_DEBUG_FS */
62
-	dev_dbg(kbdev->dev, "r: reg %08x val %08x", offset, val);
63
-
64
-	return val;
65
-}
66
-
67
-KBASE_EXPORT_TEST_API(kbase_reg_read);
68
-
6931 bool kbase_is_gpu_removed(struct kbase_device *kbdev)
7032 {
7133 	u32 val;
....@@ -76,7 +38,145 @@
7638 }
7739 #endif /* !IS_ENABLED(CONFIG_MALI_BIFROST_NO_MALI) */
7840 
79
-void kbase_gpu_start_cache_clean_nolock(struct kbase_device *kbdev)
41
+static int busy_wait_on_irq(struct kbase_device *kbdev, u32 irq_bit)
42
+{
43
+	char *irq_flag_name;
44
+	/* Previously MMU-AS command was used for L2 cache flush on page-table update.
45
+	 * And we're using the same max-loops count for GPU command, because amount of
46
+	 * L2 cache flush overhead are same between them.
47
+	 */
48
+	unsigned int max_loops = KBASE_AS_INACTIVE_MAX_LOOPS;
49
+
50
+	/* Wait for the GPU cache clean operation to complete */
51
+	while (--max_loops &&
52
+	       !(kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT)) & irq_bit)) {
53
+		;
54
+	}
55
+
56
+	/* reset gpu if time-out occurred */
57
+	if (max_loops == 0) {
58
+		switch (irq_bit) {
59
+		case CLEAN_CACHES_COMPLETED:
60
+			irq_flag_name = "CLEAN_CACHES_COMPLETED";
61
+			break;
62
+		case FLUSH_PA_RANGE_COMPLETED:
63
+			irq_flag_name = "FLUSH_PA_RANGE_COMPLETED";
64
+			break;
65
+		default:
66
+			irq_flag_name = "UNKNOWN";
67
+			break;
68
+		}
69
+
70
+		dev_err(kbdev->dev,
71
+			"Stuck waiting on %s bit, might be caused by slow/unstable GPU clock or possible faulty FPGA connector\n",
72
+			irq_flag_name);
73
+
74
+		if (kbase_prepare_to_reset_gpu_locked(kbdev, RESET_FLAGS_NONE))
75
+			kbase_reset_gpu_locked(kbdev);
76
+		return -EBUSY;
77
+	}
78
+
79
+	/* Clear the interrupt bit. */
80
+	KBASE_KTRACE_ADD(kbdev, CORE_GPU_IRQ_CLEAR, NULL, irq_bit);
81
+	kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), irq_bit);
82
+
83
+	return 0;
84
+}
85
+
86
+#if MALI_USE_CSF
87
+#define U64_LO_MASK ((1ULL << 32) - 1)
88
+#define U64_HI_MASK (~U64_LO_MASK)
89
+
90
+int kbase_gpu_cache_flush_pa_range_and_busy_wait(struct kbase_device *kbdev, phys_addr_t phys,
91
+						 size_t nr_bytes, u32 flush_op)
92
+{
93
+	u64 start_pa, end_pa;
94
+	int ret = 0;
95
+
96
+	lockdep_assert_held(&kbdev->hwaccess_lock);
97
+
98
+	/* 1. Clear the interrupt FLUSH_PA_RANGE_COMPLETED bit. */
99
+	kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), FLUSH_PA_RANGE_COMPLETED);
100
+
101
+	/* 2. Issue GPU_CONTROL.COMMAND.FLUSH_PA_RANGE operation. */
102
+	start_pa = phys;
103
+	end_pa = start_pa + nr_bytes - 1;
104
+
105
+	kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND_ARG0_LO), start_pa & U64_LO_MASK);
106
+	kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND_ARG0_HI),
107
+			(start_pa & U64_HI_MASK) >> 32);
108
+	kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND_ARG1_LO), end_pa & U64_LO_MASK);
109
+	kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND_ARG1_HI), (end_pa & U64_HI_MASK) >> 32);
110
+	kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), flush_op);
111
+
112
+	/* 3. Busy-wait irq status to be enabled. */
113
+	ret = busy_wait_on_irq(kbdev, (u32)FLUSH_PA_RANGE_COMPLETED);
114
+
115
+	return ret;
116
+}
117
+#endif /* MALI_USE_CSF */
118
+
119
+int kbase_gpu_cache_flush_and_busy_wait(struct kbase_device *kbdev,
120
+					u32 flush_op)
121
+{
122
+	int need_to_wake_up = 0;
123
+	int ret = 0;
124
+
125
+	/* hwaccess_lock must be held to avoid any sync issue with
126
+	 * kbase_gpu_start_cache_clean() / kbase_clean_caches_done()
127
+	 */
128
+	lockdep_assert_held(&kbdev->hwaccess_lock);
129
+
130
+	/* 1. Check if kbdev->cache_clean_in_progress is set.
131
+	 *    If it is set, it means there are threads waiting for
132
+	 *    CLEAN_CACHES_COMPLETED irq to be raised and that the
133
+	 *    corresponding irq mask bit is set.
134
+	 *    We'll clear the irq mask bit and busy-wait for the cache
135
+	 *    clean operation to complete before submitting the cache
136
+	 *    clean command required after the GPU page table update.
137
+	 *    Pended flush commands will be merged to requested command.
138
+	 */
139
+	if (kbdev->cache_clean_in_progress) {
140
+		/* disable irq first */
141
+		u32 irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK));
142
+		kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK),
143
+				irq_mask & ~CLEAN_CACHES_COMPLETED);
144
+
145
+		/* busy wait irq status to be enabled */
146
+		ret = busy_wait_on_irq(kbdev, (u32)CLEAN_CACHES_COMPLETED);
147
+		if (ret)
148
+			return ret;
149
+
150
+		/* merge pended command if there's any */
151
+		flush_op = GPU_COMMAND_FLUSH_CACHE_MERGE(
152
+			kbdev->cache_clean_queued, flush_op);
153
+
154
+		/* enable wake up notify flag */
155
+		need_to_wake_up = 1;
156
+	} else {
157
+		/* Clear the interrupt CLEAN_CACHES_COMPLETED bit. */
158
+		kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR),
159
+				CLEAN_CACHES_COMPLETED);
160
+	}
161
+
162
+	/* 2. Issue GPU_CONTROL.COMMAND.FLUSH_CACHE operation. */
163
+	KBASE_KTRACE_ADD(kbdev, CORE_GPU_CLEAN_INV_CACHES, NULL, flush_op);
164
+	kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), flush_op);
165
+
166
+	/* 3. Busy-wait irq status to be enabled. */
167
+	ret = busy_wait_on_irq(kbdev, (u32)CLEAN_CACHES_COMPLETED);
168
+	if (ret)
169
+		return ret;
170
+
171
+	/* 4. Wake-up blocked threads when there is any. */
172
+	if (need_to_wake_up)
173
+		kbase_gpu_cache_clean_wait_complete(kbdev);
174
+
175
+	return ret;
176
+}
177
+
178
+void kbase_gpu_start_cache_clean_nolock(struct kbase_device *kbdev,
179
+					u32 flush_op)
80180 {
81181 	u32 irq_mask;
82182 
....@@ -85,10 +185,11 @@
85185 	if (kbdev->cache_clean_in_progress) {
86186 		/* If this is called while another clean is in progress, we
87187 		 * can't rely on the current one to flush any new changes in
88
-		 * the cache. Instead, trigger another cache clean immediately
89
-		 * after this one finishes.
188
+		 * the cache. Instead, accumulate all cache clean operations
189
+		 * and trigger that immediately after this one finishes.
90190 		 */
91
-		kbdev->cache_clean_queued = true;
191
+		kbdev->cache_clean_queued = GPU_COMMAND_FLUSH_CACHE_MERGE(
192
+			kbdev->cache_clean_queued, flush_op);
92193 		return;
93194 	}
94195 
....@@ -97,19 +198,18 @@
97198 	kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK),
98199 				irq_mask | CLEAN_CACHES_COMPLETED);
99200 
100
-	KBASE_KTRACE_ADD(kbdev, CORE_GPU_CLEAN_INV_CACHES, NULL, 0);
101
-	kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
102
-					GPU_COMMAND_CLEAN_INV_CACHES);
201
+	KBASE_KTRACE_ADD(kbdev, CORE_GPU_CLEAN_INV_CACHES, NULL, flush_op);
202
+	kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), flush_op);
103203 
104204 	kbdev->cache_clean_in_progress = true;
105205 }
106206 
107
-void kbase_gpu_start_cache_clean(struct kbase_device *kbdev)
207
+void kbase_gpu_start_cache_clean(struct kbase_device *kbdev, u32 flush_op)
108208 {
109209 	unsigned long flags;
110210 
111211 	spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
112
-	kbase_gpu_start_cache_clean_nolock(kbdev);
212
+	kbase_gpu_start_cache_clean_nolock(kbdev, flush_op);
113213 	spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
114214 }
115215 
....@@ -117,7 +217,7 @@
117217 {
118218 	lockdep_assert_held(&kbdev->hwaccess_lock);
119219 
120
-	kbdev->cache_clean_queued = false;
220
+	kbdev->cache_clean_queued = 0;
121221 	kbdev->cache_clean_in_progress = false;
122222 	wake_up(&kbdev->cache_clean_wait);
123223 }
....@@ -129,19 +229,28 @@
129229 
130230 	spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
131231 
132
-	if (kbdev->cache_clean_queued) {
133
-		kbdev->cache_clean_queued = false;
232
+	if (kbdev->cache_clean_in_progress) {
233
+		/* Clear the interrupt CLEAN_CACHES_COMPLETED bit if set.
234
+		 * It might have already been done by kbase_gpu_cache_flush_and_busy_wait.
235
+		 */
236
+		KBASE_KTRACE_ADD(kbdev, CORE_GPU_IRQ_CLEAR, NULL, CLEAN_CACHES_COMPLETED);
237
+		kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), CLEAN_CACHES_COMPLETED);
134238 
135
-		KBASE_KTRACE_ADD(kbdev, CORE_GPU_CLEAN_INV_CACHES, NULL, 0);
136
-		kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
137
-				GPU_COMMAND_CLEAN_INV_CACHES);
138
-	} else {
139
-		/* Disable interrupt */
140
-		irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK));
141
-		kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK),
142
-				irq_mask & ~CLEAN_CACHES_COMPLETED);
239
+		if (kbdev->cache_clean_queued) {
240
+			u32 pended_flush_op = kbdev->cache_clean_queued;
143241 
144
-		kbase_gpu_cache_clean_wait_complete(kbdev);
242
+			kbdev->cache_clean_queued = 0;
243
+
244
+			KBASE_KTRACE_ADD(kbdev, CORE_GPU_CLEAN_INV_CACHES, NULL, pended_flush_op);
245
+			kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), pended_flush_op);
246
+		} else {
247
+			/* Disable interrupt */
248
+			irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK));
249
+			kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK),
250
+					irq_mask & ~CLEAN_CACHES_COMPLETED);
251
+
252
+			kbase_gpu_cache_clean_wait_complete(kbdev);
253
+		}
145254 	}
146255 
147256 	spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);