forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2024-05-10 cde9070d9970eef1f7ec2360586c802a16230ad8 
 kernel/arch/arm64/mm/cache.S
....@@ -1,20 +1,9 @@
1
+/* SPDX-License-Identifier: GPL-2.0-only */
12 /*
23  * Cache maintenance
34  *
45  * Copyright (C) 2001 Deep Blue Solutions Ltd.
56  * Copyright (C) 2012 ARM Ltd.
6
- *
7
- * This program is free software; you can redistribute it and/or modify
8
- * it under the terms of the GNU General Public License version 2 as
9
- * published by the Free Software Foundation.
10
- *
11
- * This program is distributed in the hope that it will be useful,
12
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
13
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14
- * GNU General Public License for more details.
15
- *
16
- * You should have received a copy of the GNU General Public License
17
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
187  */
198 
209 #include <linux/errno.h>
....@@ -26,79 +15,6 @@
2615 #include <asm/asm-uaccess.h>
2716 
2817 /*
29
- *	__flush_dcache_all()
30
- *
31
- *	Flush the whole D-cache.
32
- *
33
- *	Corrupted registers: x0-x7, x9-x11
34
- */
35
-ENTRY(__flush_dcache_all)
36
-	dmb	sy				// ensure ordering with previous memory accesses
37
-	mrs	x0, clidr_el1			// read clidr
38
-	and	x3, x0, #0x7000000		// extract loc from clidr
39
-	lsr	x3, x3, #23			// left align loc bit field
40
-	cbz	x3, finished			// if loc is 0, then no need to clean
41
-	mov	x10, #0				// start clean at cache level 0
42
-loop1:
43
-	add	x2, x10, x10, lsr #1		// work out 3x current cache level
44
-	lsr	x1, x0, x2			// extract cache type bits from clidr
45
-	and	x1, x1, #7			// mask of the bits for current cache only
46
-	cmp	x1, #2				// see what cache we have at this level
47
-	b.lt	skip				// skip if no cache, or just i-cache
48
-	save_and_disable_irqs x9		// make CSSELR and CCSIDR access atomic
49
-	msr	csselr_el1, x10			// select current cache level in csselr
50
-	isb					// isb to sych the new cssr&csidr
51
-	mrs	x1, ccsidr_el1			// read the new ccsidr
52
-	restore_irqs x9
53
-	and	x2, x1, #7			// extract the length of the cache lines
54
-	add	x2, x2, #4			// add 4 (line length offset)
55
-	mov	x4, #0x3ff
56
-	and	x4, x4, x1, lsr #3		// find maximum number on the way size
57
-	clz	w5, w4				// find bit position of way size increment
58
-	mov	x7, #0x7fff
59
-	and	x7, x7, x1, lsr #13		// extract max number of the index size
60
-loop2:
61
-	mov	x9, x4				// create working copy of max way size
62
-loop3:
63
-	lsl	x6, x9, x5
64
-	orr	x11, x10, x6			// factor way and cache number into x11
65
-	lsl	x6, x7, x2
66
-	orr	x11, x11, x6			// factor index number into x11
67
-	dc	cisw, x11			// clean & invalidate by set/way
68
-	subs	x9, x9, #1			// decrement the way
69
-	b.ge	loop3
70
-	subs	x7, x7, #1			// decrement the index
71
-	b.ge	loop2
72
-skip:
73
-	add	x10, x10, #2			// increment cache number
74
-	cmp	x3, x10
75
-	b.gt	loop1
76
-finished:
77
-	mov	x10, #0				// swith back to cache level 0
78
-	msr	csselr_el1, x10			// select current cache level in csselr
79
-	dsb	sy
80
-	isb
81
-	ret
82
-ENDPROC(__flush_dcache_all)
83
-
84
-/*
85
- *	flush_cache_all()
86
- *
87
- *	Flush the entire cache system.  The data cache flush is now achieved
88
- *	using atomic clean / invalidates working outwards from L1 cache. This
89
- *	is done using Set/Way based cache maintenance instructions.  The
90
- *	instruction cache can still be invalidated back to the point of
91
- *	unification in a single instruction.
92
- */
93
-ENTRY(flush_cache_all)
94
-	mov	x12, lr
95
-	bl	__flush_dcache_all
96
-	mov	x0, #0
97
-	ic	ialluis				// I+BTB cache invalidate
98
-	ret	x12
99
-ENDPROC(flush_cache_all)
100
-
101
-/*
10218  *	flush_icache_range(start,end)
10319  *
10420  *	Ensure that the I and D caches are coherent within specified region.
....@@ -108,7 +24,7 @@
10824  *	- start   - virtual start address of region
10925  *	- end     - virtual end address of region
11026  */
111
-ENTRY(__flush_icache_range)
27
+SYM_FUNC_START(__flush_icache_range)
11228 	/* FALLTHROUGH */
11329 
11430 /*
....@@ -121,7 +37,7 @@
12137  *	- start   - virtual start address of region
12238  *	- end     - virtual end address of region
12339  */
124
-ENTRY(__flush_cache_user_range)
40
+SYM_FUNC_START(__flush_cache_user_range)
12541 	uaccess_ttbr0_enable x2, x3, x4
12642 alternative_if ARM64_HAS_CACHE_IDC
12743 	dsb	ishst
....@@ -150,8 +66,8 @@
15066 9:
15167 	mov	x0, #-EFAULT
15268 	b	1b
153
-ENDPROC(__flush_icache_range)
154
-ENDPROC(__flush_cache_user_range)
69
+SYM_FUNC_END(__flush_icache_range)
70
+SYM_FUNC_END(__flush_cache_user_range)
15571 
15672 /*
15773  *	invalidate_icache_range(start,end)
....@@ -161,7 +77,7 @@
16177  *	- start   - virtual start address of region
16278  *	- end     - virtual end address of region
16379  */
164
-ENTRY(invalidate_icache_range)
80
+SYM_FUNC_START(invalidate_icache_range)
16581 alternative_if ARM64_HAS_CACHE_DIC
16682 	mov	x0, xzr
16783 	isb
....@@ -178,7 +94,7 @@
17894 2:
17995 	mov	x0, #-EFAULT
18096 	b	1b
181
-ENDPROC(invalidate_icache_range)
97
+SYM_FUNC_END(invalidate_icache_range)
18298 
18399 /*
184100  *	__flush_dcache_area(kaddr, size)
....@@ -189,10 +105,10 @@
189105  *	- kaddr   - kernel address
190106  *	- size    - size in question
191107  */
192
-ENTRY(__flush_dcache_area)
108
+SYM_FUNC_START_PI(__flush_dcache_area)
193109 	dcache_by_line_op civac, sy, x0, x1, x2, x3
194110 	ret
195
-ENDPIPROC(__flush_dcache_area)
111
+SYM_FUNC_END_PI(__flush_dcache_area)
196112 
197113 /*
198114  *	__clean_dcache_area_pou(kaddr, size)
....@@ -203,14 +119,14 @@
203119  *	- kaddr   - kernel address
204120  *	- size    - size in question
205121  */
206
-ENTRY(__clean_dcache_area_pou)
122
+SYM_FUNC_START(__clean_dcache_area_pou)
207123 alternative_if ARM64_HAS_CACHE_IDC
208124 	dsb	ishst
209125 	ret
210126 alternative_else_nop_endif
211127 	dcache_by_line_op cvau, ish, x0, x1, x2, x3
212128 	ret
213
-ENDPROC(__clean_dcache_area_pou)
129
+SYM_FUNC_END(__clean_dcache_area_pou)
214130 
215131 /*
216132  *	__inval_dcache_area(kaddr, size)
....@@ -222,7 +138,8 @@
222138  *	- kaddr   - kernel address
223139  *	- size    - size in question
224140  */
225
-ENTRY(__inval_dcache_area)
141
+SYM_FUNC_START_LOCAL(__dma_inv_area)
142
+SYM_FUNC_START_PI(__inval_dcache_area)
226143 	/* FALLTHROUGH */
227144 
228145 /*
....@@ -230,7 +147,6 @@
230147  *	- start   - virtual start address of region
231148  *	- size    - size in question
232149  */
233
-ENTRY(__dma_inv_area)
234150 	add	x1, x1, x0
235151 	dcache_line_size x2, x3
236152 	sub	x3, x2, #1
....@@ -249,8 +165,8 @@
249165 	b.lo	2b
250166 	dsb	sy
251167 	ret
252
-ENDPIPROC(__inval_dcache_area)
253
-ENDPROC(__dma_inv_area)
168
+SYM_FUNC_END_PI(__inval_dcache_area)
169
+SYM_FUNC_END(__dma_inv_area)
254170 
255171 /*
256172  *	__clean_dcache_area_poc(kaddr, size)
....@@ -261,7 +177,8 @@
261177  *	- kaddr   - kernel address
262178  *	- size    - size in question
263179  */
264
-ENTRY(__clean_dcache_area_poc)
180
+SYM_FUNC_START_LOCAL(__dma_clean_area)
181
+SYM_FUNC_START_PI(__clean_dcache_area_poc)
265182 	/* FALLTHROUGH */
266183 
267184 /*
....@@ -269,11 +186,10 @@
269186  *	- start   - virtual start address of region
270187  *	- size    - size in question
271188  */
272
-ENTRY(__dma_clean_area)
273189 	dcache_by_line_op cvac, sy, x0, x1, x2, x3
274190 	ret
275
-ENDPIPROC(__clean_dcache_area_poc)
276
-ENDPROC(__dma_clean_area)
191
+SYM_FUNC_END_PI(__clean_dcache_area_poc)
192
+SYM_FUNC_END(__dma_clean_area)
277193 
278194 /*
279195  *	__clean_dcache_area_pop(kaddr, size)
....@@ -284,13 +200,13 @@
284200  *	- kaddr   - kernel address
285201  *	- size    - size in question
286202  */
287
-ENTRY(__clean_dcache_area_pop)
203
+SYM_FUNC_START_PI(__clean_dcache_area_pop)
288204 	alternative_if_not ARM64_HAS_DCPOP
289205 	b	__clean_dcache_area_poc
290206 	alternative_else_nop_endif
291207 	dcache_by_line_op cvap, sy, x0, x1, x2, x3
292208 	ret
293
-ENDPIPROC(__clean_dcache_area_pop)
209
+SYM_FUNC_END_PI(__clean_dcache_area_pop)
294210 
295211 /*
296212  *	__dma_flush_area(start, size)
....@@ -300,10 +216,10 @@
300216  *	- start   - virtual start address of region
301217  *	- size    - size in question
302218  */
303
-ENTRY(__dma_flush_area)
219
+SYM_FUNC_START_PI(__dma_flush_area)
304220 	dcache_by_line_op civac, sy, x0, x1, x2, x3
305221 	ret
306
-ENDPIPROC(__dma_flush_area)
222
+SYM_FUNC_END_PI(__dma_flush_area)
307223 
308224 /*
309225  *	__dma_map_area(start, size, dir)
....@@ -311,11 +227,11 @@
311227  *	- size	- size of region
312228  *	- dir	- DMA direction
313229  */
314
-ENTRY(__dma_map_area)
230
+SYM_FUNC_START_PI(__dma_map_area)
315231 	cmp	w2, #DMA_FROM_DEVICE
316
-	b.eq	__dma_inv_area
232
+	b.eq	__dma_flush_area
317233 	b	__dma_clean_area
318
-ENDPIPROC(__dma_map_area)
234
+SYM_FUNC_END_PI(__dma_map_area)
319235 
320236 /*
321237  *	__dma_unmap_area(start, size, dir)
....@@ -323,8 +239,8 @@
323239  *	- size	- size of region
324240  *	- dir	- DMA direction
325241  */
326
-ENTRY(__dma_unmap_area)
242
+SYM_FUNC_START_PI(__dma_unmap_area)
327243 	cmp	w2, #DMA_TO_DEVICE
328244 	b.ne	__dma_inv_area
329245 	ret
330
-ENDPIPROC(__dma_unmap_area)
246
+SYM_FUNC_END_PI(__dma_unmap_area)