forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2024-02-20 102a0743326a03cd1a1202ceda21e175b7d3575c 
 kernel/arch/nios2/mm/tlb.c
....@@ -16,16 +16,11 @@
1616 
1717 #include <asm/tlb.h>
1818 #include <asm/mmu_context.h>
19
-#include <asm/pgtable.h>
2019 #include <asm/cpuinfo.h>
2120 
2221 #define TLB_INDEX_MASK		\
2322 	((((1UL << (cpuinfo.tlb_ptr_sz - cpuinfo.tlb_num_ways_log2))) - 1) \
2423 		<< PAGE_SHIFT)
25
-
26
-/* Used as illegal PHYS_ADDR for TLB mappings
27
- */
28
-#define MAX_PHYS_ADDR 0
2924 
3025 static void get_misc_and_pid(unsigned long *misc, unsigned long *pid)
3126 {
....@@ -35,27 +30,22 @@
3530 }
3631 
3732 /*
38
- * All entries common to a mm share an asid.  To effectively flush these
39
- * entries, we just bump the asid.
33
+ * This provides a PTEADDR value for addr that will cause a TLB miss
34
+ * (fast TLB miss). TLB invalidation replaces entries with this value.
4035  */
41
-void flush_tlb_mm(struct mm_struct *mm)
36
+static unsigned long pteaddr_invalid(unsigned long addr)
4237 {
43
-	if (current->mm == mm)
44
-		flush_tlb_all();
45
-	else
46
-		memset(&mm->context, 0, sizeof(mm_context_t));
38
+	return ((addr | 0xC0000000UL) >> PAGE_SHIFT) << 2;
4739 }
4840 
4941 /*
5042  * This one is only used for pages with the global bit set so we don't care
5143  * much about the ASID.
5244  */
53
-void flush_tlb_one_pid(unsigned long addr, unsigned long mmu_pid)
45
+static void replace_tlb_one_pid(unsigned long addr, unsigned long mmu_pid, unsigned long tlbacc)
5446 {
5547 	unsigned int way;
5648 	unsigned long org_misc, pid_misc;
57
-
58
-	pr_debug("Flush tlb-entry for vaddr=%#lx\n", addr);
5949 
6050 	/* remember pid/way until we return. */
6151 	get_misc_and_pid(&org_misc, &pid_misc);
....@@ -67,28 +57,46 @@
6757 		unsigned long tlbmisc;
6858 		unsigned long pid;
6959 
70
-		tlbmisc = pid_misc | TLBMISC_RD | (way << TLBMISC_WAY_SHIFT);
60
+		tlbmisc = TLBMISC_RD | (way << TLBMISC_WAY_SHIFT);
7161 		WRCTL(CTL_TLBMISC, tlbmisc);
62
+
7263 		pteaddr = RDCTL(CTL_PTEADDR);
64
+		if (((pteaddr >> 2) & 0xfffff) != (addr >> PAGE_SHIFT))
65
+			continue;
66
+
7367 		tlbmisc = RDCTL(CTL_TLBMISC);
7468 		pid = (tlbmisc >> TLBMISC_PID_SHIFT) & TLBMISC_PID_MASK;
75
-		if (((((pteaddr >> 2) & 0xfffff)) == (addr >> PAGE_SHIFT)) &&
76
-				pid == mmu_pid) {
77
-			unsigned long vaddr = CONFIG_NIOS2_IO_REGION_BASE +
78
-				((PAGE_SIZE * cpuinfo.tlb_num_lines) * way) +
79
-				(addr & TLB_INDEX_MASK);
80
-			pr_debug("Flush entry by writing %#lx way=%dl pid=%ld\n",
81
-				vaddr, way, (pid_misc >> TLBMISC_PID_SHIFT));
69
+		if (pid != mmu_pid)
70
+			continue;
8271 
83
-			WRCTL(CTL_PTEADDR, (vaddr >> 12) << 2);
84
-			tlbmisc = pid_misc | TLBMISC_WE |
85
-				(way << TLBMISC_WAY_SHIFT);
86
-			WRCTL(CTL_TLBMISC, tlbmisc);
87
-			WRCTL(CTL_TLBACC, (MAX_PHYS_ADDR >> PAGE_SHIFT));
88
-		}
72
+		tlbmisc = (mmu_pid << TLBMISC_PID_SHIFT) | TLBMISC_WE |
73
+			  (way << TLBMISC_WAY_SHIFT);
74
+		WRCTL(CTL_TLBMISC, tlbmisc);
75
+		if (tlbacc == 0)
76
+			WRCTL(CTL_PTEADDR, pteaddr_invalid(addr));
77
+		WRCTL(CTL_TLBACC, tlbacc);
78
+		/*
79
+		 * There should be only a single entry that maps a
80
+		 * particular {address,pid} so break after a match.
81
+		 */
82
+		break;
8983 	}
9084 
9185 	WRCTL(CTL_TLBMISC, org_misc);
86
+}
87
+
88
+static void flush_tlb_one_pid(unsigned long addr, unsigned long mmu_pid)
89
+{
90
+	pr_debug("Flush tlb-entry for vaddr=%#lx\n", addr);
91
+
92
+	replace_tlb_one_pid(addr, mmu_pid, 0);
93
+}
94
+
95
+static void reload_tlb_one_pid(unsigned long addr, unsigned long mmu_pid, pte_t pte)
96
+{
97
+	pr_debug("Reload tlb-entry for vaddr=%#lx\n", addr);
98
+
99
+	replace_tlb_one_pid(addr, mmu_pid, pte_val(pte));
92100 }
93101 
94102 void flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
....@@ -102,19 +110,18 @@
102110 	}
103111 }
104112 
105
-void flush_tlb_kernel_range(unsigned long start, unsigned long end)
113
+void reload_tlb_page(struct vm_area_struct *vma, unsigned long addr, pte_t pte)
106114 {
107
-	while (start < end) {
108
-		flush_tlb_one(start);
109
-		start += PAGE_SIZE;
110
-	}
115
+	unsigned long mmu_pid = get_pid_from_context(&vma->vm_mm->context);
116
+
117
+	reload_tlb_one_pid(addr, mmu_pid, pte);
111118 }
112119 
113120 /*
114121  * This one is only used for pages with the global bit set so we don't care
115122  * much about the ASID.
116123  */
117
-void flush_tlb_one(unsigned long addr)
124
+static void flush_tlb_one(unsigned long addr)
118125 {
119126 	unsigned int way;
120127 	unsigned long org_misc, pid_misc;
....@@ -130,28 +137,31 @@
130137 		unsigned long pteaddr;
131138 		unsigned long tlbmisc;
132139 
133
-		tlbmisc = pid_misc | TLBMISC_RD | (way << TLBMISC_WAY_SHIFT);
140
+		tlbmisc = TLBMISC_RD | (way << TLBMISC_WAY_SHIFT);
134141 		WRCTL(CTL_TLBMISC, tlbmisc);
142
+
135143 		pteaddr = RDCTL(CTL_PTEADDR);
136
-		tlbmisc = RDCTL(CTL_TLBMISC);
144
+		if (((pteaddr >> 2) & 0xfffff) != (addr >> PAGE_SHIFT))
145
+			continue;
137146 
138
-		if ((((pteaddr >> 2) & 0xfffff)) == (addr >> PAGE_SHIFT)) {
139
-			unsigned long vaddr = CONFIG_NIOS2_IO_REGION_BASE +
140
-				((PAGE_SIZE * cpuinfo.tlb_num_lines) * way) +
141
-				(addr & TLB_INDEX_MASK);
147
+		pr_debug("Flush entry by writing way=%dl pid=%ld\n",
148
+			 way, (pid_misc >> TLBMISC_PID_SHIFT));
142149 
143
-			pr_debug("Flush entry by writing %#lx way=%dl pid=%ld\n",
144
-				vaddr, way, (pid_misc >> TLBMISC_PID_SHIFT));
145
-
146
-			tlbmisc = pid_misc | TLBMISC_WE |
147
-				(way << TLBMISC_WAY_SHIFT);
148
-			WRCTL(CTL_PTEADDR, (vaddr >> 12) << 2);
149
-			WRCTL(CTL_TLBMISC, tlbmisc);
150
-			WRCTL(CTL_TLBACC, (MAX_PHYS_ADDR >> PAGE_SHIFT));
151
-		}
150
+		tlbmisc = TLBMISC_WE | (way << TLBMISC_WAY_SHIFT);
151
+		WRCTL(CTL_TLBMISC, tlbmisc);
152
+		WRCTL(CTL_PTEADDR, pteaddr_invalid(addr));
153
+		WRCTL(CTL_TLBACC, 0);
152154 	}
153155 
154156 	WRCTL(CTL_TLBMISC, org_misc);
157
+}
158
+
159
+void flush_tlb_kernel_range(unsigned long start, unsigned long end)
160
+{
161
+	while (start < end) {
162
+		flush_tlb_one(start);
163
+		start += PAGE_SIZE;
164
+	}
155165 }
156166 
157167 void dump_tlb_line(unsigned long line)
....@@ -177,7 +187,7 @@
177187 		tlbmisc = RDCTL(CTL_TLBMISC);
178188 		tlbacc = RDCTL(CTL_TLBACC);
179189 
180
-		if ((tlbacc << PAGE_SHIFT) != (MAX_PHYS_ADDR & PAGE_MASK)) {
190
+		if ((tlbacc << PAGE_SHIFT) != 0) {
181191 			pr_debug("-- way:%02x vpn:0x%08lx phys:0x%08lx pid:0x%02lx flags:%c%c%c%c%c\n",
182192 				way,
183193 				(pteaddr << (PAGE_SHIFT-2)),
....@@ -203,8 +213,9 @@
203213 		dump_tlb_line(i);
204214 }
205215 
206
-void flush_tlb_pid(unsigned long pid)
216
+void flush_tlb_pid(unsigned long mmu_pid)
207217 {
218
+	unsigned long addr = 0;
208219 	unsigned int line;
209220 	unsigned int way;
210221 	unsigned long org_misc, pid_misc;
....@@ -213,55 +224,65 @@
213224 	get_misc_and_pid(&org_misc, &pid_misc);
214225 
215226 	for (line = 0; line < cpuinfo.tlb_num_lines; line++) {
216
-		WRCTL(CTL_PTEADDR, line << 2);
227
+		WRCTL(CTL_PTEADDR, pteaddr_invalid(addr));
217228 
218229 		for (way = 0; way < cpuinfo.tlb_num_ways; way++) {
219
-			unsigned long pteaddr;
220230 			unsigned long tlbmisc;
221
-			unsigned long tlbacc;
231
+			unsigned long pid;
222232 
223
-			tlbmisc = pid_misc | TLBMISC_RD |
224
-				(way << TLBMISC_WAY_SHIFT);
233
+			tlbmisc = TLBMISC_RD | (way << TLBMISC_WAY_SHIFT);
225234 			WRCTL(CTL_TLBMISC, tlbmisc);
226
-			pteaddr = RDCTL(CTL_PTEADDR);
227235 			tlbmisc = RDCTL(CTL_TLBMISC);
228
-			tlbacc = RDCTL(CTL_TLBACC);
236
+			pid = (tlbmisc >> TLBMISC_PID_SHIFT) & TLBMISC_PID_MASK;
237
+			if (pid != mmu_pid)
238
+				continue;
229239 
230
-			if (((tlbmisc>>TLBMISC_PID_SHIFT) & TLBMISC_PID_MASK)
231
-				== pid) {
232
-				tlbmisc = pid_misc | TLBMISC_WE |
233
-					(way << TLBMISC_WAY_SHIFT);
234
-				WRCTL(CTL_TLBMISC, tlbmisc);
235
-				WRCTL(CTL_TLBACC,
236
-					(MAX_PHYS_ADDR >> PAGE_SHIFT));
237
-			}
240
+			tlbmisc = TLBMISC_WE | (way << TLBMISC_WAY_SHIFT);
241
+			WRCTL(CTL_TLBMISC, tlbmisc);
242
+			WRCTL(CTL_TLBACC, 0);
238243 		}
239244 
240
-		WRCTL(CTL_TLBMISC, org_misc);
245
+		addr += PAGE_SIZE;
246
+	}
247
+
248
+	WRCTL(CTL_TLBMISC, org_misc);
249
+}
250
+
251
+/*
252
+ * All entries common to a mm share an asid.  To effectively flush these
253
+ * entries, we just bump the asid.
254
+ */
255
+void flush_tlb_mm(struct mm_struct *mm)
256
+{
257
+	if (current->mm == mm) {
258
+		unsigned long mmu_pid = get_pid_from_context(&mm->context);
259
+		flush_tlb_pid(mmu_pid);
260
+	} else {
261
+		memset(&mm->context, 0, sizeof(mm_context_t));
241262 	}
242263 }
243264 
244265 void flush_tlb_all(void)
245266 {
246
-	int i;
247
-	unsigned long vaddr = CONFIG_NIOS2_IO_REGION_BASE;
267
+	unsigned long addr = 0;
268
+	unsigned int line;
248269 	unsigned int way;
249
-	unsigned long org_misc, pid_misc, tlbmisc;
270
+	unsigned long org_misc, pid_misc;
250271 
251272 	/* remember pid/way until we return */
252273 	get_misc_and_pid(&org_misc, &pid_misc);
253
-	pid_misc |= TLBMISC_WE;
274
+
275
+	/* Start at way 0, way is auto-incremented after each TLBACC write */
276
+	WRCTL(CTL_TLBMISC, TLBMISC_WE);
254277 
255278 	/* Map each TLB entry to physcal address 0 with no-access and a
256279 	   bad ptbase */
257
-	for (way = 0; way < cpuinfo.tlb_num_ways; way++) {
258
-		tlbmisc = pid_misc | (way << TLBMISC_WAY_SHIFT);
259
-		for (i = 0; i < cpuinfo.tlb_num_lines; i++) {
260
-			WRCTL(CTL_PTEADDR, ((vaddr) >> PAGE_SHIFT) << 2);
261
-			WRCTL(CTL_TLBMISC, tlbmisc);
262
-			WRCTL(CTL_TLBACC, (MAX_PHYS_ADDR >> PAGE_SHIFT));
263
-			vaddr += 1UL << 12;
264
-		}
280
+	for (line = 0; line < cpuinfo.tlb_num_lines; line++) {
281
+		WRCTL(CTL_PTEADDR, pteaddr_invalid(addr));
282
+		for (way = 0; way < cpuinfo.tlb_num_ways; way++)
283
+			WRCTL(CTL_TLBACC, 0);
284
+
285
+		addr += PAGE_SIZE;
265286 	}
266287 
267288 	/* restore pid/way */
....@@ -270,6 +291,10 @@
270291 
271292 void set_mmu_pid(unsigned long pid)
272293 {
273
-	WRCTL(CTL_TLBMISC, (RDCTL(CTL_TLBMISC) & TLBMISC_WAY) |
274
-		((pid & TLBMISC_PID_MASK) << TLBMISC_PID_SHIFT));
294
+	unsigned long tlbmisc;
295
+
296
+	tlbmisc = RDCTL(CTL_TLBMISC);
297
+	tlbmisc = (tlbmisc & TLBMISC_WAY);
298
+	tlbmisc |= (pid & TLBMISC_PID_MASK) << TLBMISC_PID_SHIFT;
299
+	WRCTL(CTL_TLBMISC, tlbmisc);
275300 }