forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2024-09-20 cf4ce59b3b70238352c7f1729f0f7223214828ad 
 kernel/arch/parisc/kernel/pacache.S
....@@ -1,22 +1,9 @@
1
+/* SPDX-License-Identifier: GPL-2.0-or-later */
12 /*
23  *  PARISC TLB and cache flushing support
34  *  Copyright (C) 2000-2001 Hewlett-Packard (John Marvin)
45  *  Copyright (C) 2001 Matthew Wilcox (willy at parisc-linux.org)
56  *  Copyright (C) 2002 Richard Hirst (rhirst with parisc-linux.org)
6
- *
7
- *    This program is free software; you can redistribute it and/or modify
8
- *    it under the terms of the GNU General Public License as published by
9
- *    the Free Software Foundation; either version 2, or (at your option)
10
- *    any later version.
11
- *
12
- *    This program is distributed in the hope that it will be useful,
13
- *    but WITHOUT ANY WARRANTY; without even the implied warranty of
14
- *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15
- *    GNU General Public License for more details.
16
- *
17
- *    You should have received a copy of the GNU General Public License
18
- *    along with this program; if not, write to the Free Software
19
- *    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
207  */
218 
229 /*
....@@ -34,11 +21,12 @@
3421 
3522 #include <asm/psw.h>
3623 #include <asm/assembly.h>
37
-#include <asm/pgtable.h>
3824 #include <asm/cache.h>
3925 #include <asm/ldcw.h>
26
+#include <asm/alternative.h>
4027 #include <linux/linkage.h>
4128 #include <linux/init.h>
29
+#include <linux/pgtable.h>
4230 
4331 	.section .text.hot
4432 	.align	16
....@@ -75,7 +63,7 @@
7563 
7664 	/* Flush Instruction Tlb */
7765 
78
-	LDREG		ITLB_SID_BASE(%r1), %r20
66
+88:	LDREG		ITLB_SID_BASE(%r1), %r20
7967 	LDREG		ITLB_SID_STRIDE(%r1), %r21
8068 	LDREG		ITLB_SID_COUNT(%r1), %r22
8169 	LDREG		ITLB_OFF_BASE(%r1), %arg0
....@@ -115,6 +103,7 @@
115103 	add		%r21, %r20, %r20		/* increment space */
116104 
117105 fitdone:
106
+	ALTERNATIVE(88b, fitdone, ALT_COND_NO_SPLIT_TLB, INSN_NOP)
118107 
119108 	/* Flush Data Tlb */
120109 
....@@ -185,12 +174,21 @@
185174 
186175 2:      bv		%r0(%r2)
187176 	nop
177
+
178
+	/*
179
+	 * When running in qemu, drop whole flush_tlb_all_local function and
180
+	 * replace by one pdtlbe instruction, for which QEMU will drop all
181
+	 * local TLB entries.
182
+	 */
183
+3:	pdtlbe		%r0(%sr1,%r0)
184
+	bv,n		%r0(%r2)
185
+	ALTERNATIVE_CODE(flush_tlb_all_local, 2, ALT_COND_RUN_ON_QEMU, 3b)
188186 ENDPROC_CFI(flush_tlb_all_local)
189187 
190188 	.import cache_info,data
191189 
192190 ENTRY_CFI(flush_instruction_cache_local)
193
-	load32		cache_info, %r1
191
+88:	load32		cache_info, %r1
194192 
195193 	/* Flush Instruction Cache */
196194 
....@@ -243,6 +241,7 @@
243241 fisync:
244242 	sync
245243 	mtsm		%r22			/* restore I-bit */
244
+89:	ALTERNATIVE(88b, 89b, ALT_COND_NO_ICACHE, INSN_NOP)
246245 	bv		%r0(%r2)
247246 	nop
248247 ENDPROC_CFI(flush_instruction_cache_local)
....@@ -250,7 +249,7 @@
250249 
251250 	.import cache_info, data
252251 ENTRY_CFI(flush_data_cache_local)
253
-	load32		cache_info, %r1
252
+88:	load32		cache_info, %r1
254253 
255254 	/* Flush Data Cache */
256255 
....@@ -304,39 +303,10 @@
304303 	syncdma
305304 	sync
306305 	mtsm		%r22			/* restore I-bit */
306
+89:	ALTERNATIVE(88b, 89b, ALT_COND_NO_DCACHE, INSN_NOP)
307307 	bv		%r0(%r2)
308308 	nop
309309 ENDPROC_CFI(flush_data_cache_local)
310
-
311
-/* Macros to serialize TLB purge operations on SMP.  */
312
-
313
-	.macro	tlb_lock	la,flags,tmp
314
-#ifdef CONFIG_SMP
315
-#if __PA_LDCW_ALIGNMENT > 4
316
-	load32		pa_tlb_lock + __PA_LDCW_ALIGNMENT-1, \la
317
-	depi		0,31,__PA_LDCW_ALIGN_ORDER, \la
318
-#else
319
-	load32		pa_tlb_lock, \la
320
-#endif
321
-	rsm		PSW_SM_I,\flags
322
-1:	LDCW		0(\la),\tmp
323
-	cmpib,<>,n	0,\tmp,3f
324
-2:	ldw		0(\la),\tmp
325
-	cmpb,<>		%r0,\tmp,1b
326
-	nop
327
-	b,n		2b
328
-3:
329
-#endif
330
-	.endm
331
-
332
-	.macro	tlb_unlock	la,flags,tmp
333
-#ifdef CONFIG_SMP
334
-	ldi		1,\tmp
335
-	sync
336
-	stw		\tmp,0(\la)
337
-	mtsm		\flags
338
-#endif
339
-	.endm
340310 
341311 /* Clear page using kernel mapping.  */
342312 
....@@ -595,10 +565,10 @@
595565 	pdtlb,l		%r0(%r28)
596566 	pdtlb,l		%r0(%r29)
597567 #else
598
-	tlb_lock	%r20,%r21,%r22
599
-	pdtlb		%r0(%r28)
600
-	pdtlb		%r0(%r29)
601
-	tlb_unlock	%r20,%r21,%r22
568
+0:	pdtlb		%r0(%r28)
569
+1:	pdtlb		%r0(%r29)
570
+	ALTERNATIVE(0b, 0b+4, ALT_COND_NO_SMP, INSN_PxTLB)
571
+	ALTERNATIVE(1b, 1b+4, ALT_COND_NO_SMP, INSN_PxTLB)
602572 #endif
603573 
604574 #ifdef CONFIG_64BIT
....@@ -735,9 +705,8 @@
735705 #ifdef CONFIG_PA20
736706 	pdtlb,l		%r0(%r28)
737707 #else
738
-	tlb_lock	%r20,%r21,%r22
739
-	pdtlb		%r0(%r28)
740
-	tlb_unlock	%r20,%r21,%r22
708
+0:	pdtlb		%r0(%r28)
709
+	ALTERNATIVE(0b, 0b+4, ALT_COND_NO_SMP, INSN_PxTLB)
741710 #endif
742711 
743712 #ifdef CONFIG_64BIT
....@@ -812,12 +781,11 @@
812781 #ifdef CONFIG_PA20
813782 	pdtlb,l		%r0(%r28)
814783 #else
815
-	tlb_lock	%r20,%r21,%r22
816
-	pdtlb		%r0(%r28)
817
-	tlb_unlock	%r20,%r21,%r22
784
+0:	pdtlb		%r0(%r28)
785
+	ALTERNATIVE(0b, 0b+4, ALT_COND_NO_SMP, INSN_PxTLB)
818786 #endif
819787 
820
-	ldil		L%dcache_stride, %r1
788
+88:	ldil		L%dcache_stride, %r1
821789 	ldw		R%dcache_stride(%r1), r31
822790 
823791 #ifdef CONFIG_64BIT
....@@ -828,8 +796,7 @@
828796 	add		%r28, %r25, %r25
829797 	sub		%r25, r31, %r25
830798 
831
-
832
-1:      fdc,m		r31(%r28)
799
+1:	fdc,m		r31(%r28)
833800 	fdc,m		r31(%r28)
834801 	fdc,m		r31(%r28)
835802 	fdc,m		r31(%r28)
....@@ -844,13 +811,73 @@
844811 	fdc,m		r31(%r28)
845812 	fdc,m		r31(%r28)
846813 	fdc,m		r31(%r28)
847
-	cmpb,COND(<<)	%r28, %r25,1b
814
+	cmpb,COND(>>)	%r25, %r28, 1b /* predict taken */
848815 	fdc,m		r31(%r28)
849816 
817
+89:	ALTERNATIVE(88b, 89b, ALT_COND_NO_DCACHE, INSN_NOP)
850818 	sync
851819 	bv		%r0(%r2)
852820 	nop
853821 ENDPROC_CFI(flush_dcache_page_asm)
822
+
823
+ENTRY_CFI(purge_dcache_page_asm)
824
+	ldil		L%(TMPALIAS_MAP_START), %r28
825
+#ifdef CONFIG_64BIT
826
+#if (TMPALIAS_MAP_START >= 0x80000000)
827
+	depdi		0, 31,32, %r28		/* clear any sign extension */
828
+#endif
829
+	convert_phys_for_tlb_insert20 %r26	/* convert phys addr to tlb insert format */
830
+	depd		%r25, 63,22, %r28	/* Form aliased virtual address 'to' */
831
+	depdi		0, 63,PAGE_SHIFT, %r28	/* Clear any offset bits */
832
+#else
833
+	extrw,u		%r26, 24,25, %r26	/* convert phys addr to tlb insert format */
834
+	depw		%r25, 31,22, %r28	/* Form aliased virtual address 'to' */
835
+	depwi		0, 31,PAGE_SHIFT, %r28	/* Clear any offset bits */
836
+#endif
837
+
838
+	/* Purge any old translation */
839
+
840
+#ifdef CONFIG_PA20
841
+	pdtlb,l		%r0(%r28)
842
+#else
843
+0:	pdtlb		%r0(%r28)
844
+	ALTERNATIVE(0b, 0b+4, ALT_COND_NO_SMP, INSN_PxTLB)
845
+#endif
846
+
847
+88:	ldil		L%dcache_stride, %r1
848
+	ldw		R%dcache_stride(%r1), r31
849
+
850
+#ifdef CONFIG_64BIT
851
+	depdi,z		1, 63-PAGE_SHIFT,1, %r25
852
+#else
853
+	depwi,z		1, 31-PAGE_SHIFT,1, %r25
854
+#endif
855
+	add		%r28, %r25, %r25
856
+	sub		%r25, r31, %r25
857
+
858
+1:      pdc,m		r31(%r28)
859
+	pdc,m		r31(%r28)
860
+	pdc,m		r31(%r28)
861
+	pdc,m		r31(%r28)
862
+	pdc,m		r31(%r28)
863
+	pdc,m		r31(%r28)
864
+	pdc,m		r31(%r28)
865
+	pdc,m		r31(%r28)
866
+	pdc,m		r31(%r28)
867
+	pdc,m		r31(%r28)
868
+	pdc,m		r31(%r28)
869
+	pdc,m		r31(%r28)
870
+	pdc,m		r31(%r28)
871
+	pdc,m		r31(%r28)
872
+	pdc,m		r31(%r28)
873
+	cmpb,COND(>>)	%r25, %r28, 1b /* predict taken */
874
+	pdc,m		r31(%r28)
875
+
876
+89:	ALTERNATIVE(88b, 89b, ALT_COND_NO_DCACHE, INSN_NOP)
877
+	sync
878
+	bv		%r0(%r2)
879
+	nop
880
+ENDPROC_CFI(purge_dcache_page_asm)
854881 
855882 ENTRY_CFI(flush_icache_page_asm)
856883 	ldil		L%(TMPALIAS_MAP_START), %r28
....@@ -874,15 +901,17 @@
874901 
875902 #ifdef CONFIG_PA20
876903 	pdtlb,l		%r0(%r28)
877
-	pitlb,l         %r0(%sr4,%r28)
904
+1:	pitlb,l         %r0(%sr4,%r28)
905
+	ALTERNATIVE(1b, 1b+4, ALT_COND_NO_SPLIT_TLB, INSN_NOP)
878906 #else
879
-	tlb_lock        %r20,%r21,%r22
880
-	pdtlb		%r0(%r28)
881
-	pitlb           %r0(%sr4,%r28)
882
-	tlb_unlock      %r20,%r21,%r22
907
+0:	pdtlb		%r0(%r28)
908
+1:	pitlb           %r0(%sr4,%r28)
909
+	ALTERNATIVE(0b, 0b+4, ALT_COND_NO_SMP, INSN_PxTLB)
910
+	ALTERNATIVE(1b, 1b+4, ALT_COND_NO_SMP, INSN_PxTLB)
911
+	ALTERNATIVE(1b, 1b+4, ALT_COND_NO_SPLIT_TLB, INSN_NOP)
883912 #endif
884913 
885
-	ldil		L%icache_stride, %r1
914
+88:	ldil		L%icache_stride, %r1
886915 	ldw		R%icache_stride(%r1), %r31
887916 
888917 #ifdef CONFIG_64BIT
....@@ -892,7 +921,6 @@
892921 #endif
893922 	add		%r28, %r25, %r25
894923 	sub		%r25, %r31, %r25
895
-
896924 
897925 	/* fic only has the type 26 form on PA1.1, requiring an
898926 	 * explicit space specification, so use %sr4 */
....@@ -911,16 +939,17 @@
911939 	fic,m		%r31(%sr4,%r28)
912940 	fic,m		%r31(%sr4,%r28)
913941 	fic,m		%r31(%sr4,%r28)
914
-	cmpb,COND(<<)	%r28, %r25,1b
942
+	cmpb,COND(>>)	%r25, %r28, 1b /* predict taken */
915943 	fic,m		%r31(%sr4,%r28)
916944 
945
+89:	ALTERNATIVE(88b, 89b, ALT_COND_NO_ICACHE, INSN_NOP)
917946 	sync
918947 	bv		%r0(%r2)
919948 	nop
920949 ENDPROC_CFI(flush_icache_page_asm)
921950 
922951 ENTRY_CFI(flush_kernel_dcache_page_asm)
923
-	ldil		L%dcache_stride, %r1
952
+88:	ldil		L%dcache_stride, %r1
924953 	ldw		R%dcache_stride(%r1), %r23
925954 
926955 #ifdef CONFIG_64BIT
....@@ -930,7 +959,6 @@
930959 #endif
931960 	add		%r26, %r25, %r25
932961 	sub		%r25, %r23, %r25
933
-
934962 
935963 1:      fdc,m		%r23(%r26)
936964 	fdc,m		%r23(%r26)
....@@ -947,16 +975,17 @@
947975 	fdc,m		%r23(%r26)
948976 	fdc,m		%r23(%r26)
949977 	fdc,m		%r23(%r26)
950
-	cmpb,COND(<<)		%r26, %r25,1b
978
+	cmpb,COND(>>)	%r25, %r26, 1b /* predict taken */
951979 	fdc,m		%r23(%r26)
952980 
981
+89:	ALTERNATIVE(88b, 89b, ALT_COND_NO_DCACHE, INSN_NOP)
953982 	sync
954983 	bv		%r0(%r2)
955984 	nop
956985 ENDPROC_CFI(flush_kernel_dcache_page_asm)
957986 
958987 ENTRY_CFI(purge_kernel_dcache_page_asm)
959
-	ldil		L%dcache_stride, %r1
988
+88:	ldil		L%dcache_stride, %r1
960989 	ldw		R%dcache_stride(%r1), %r23
961990 
962991 #ifdef CONFIG_64BIT
....@@ -982,74 +1011,183 @@
9821011 	pdc,m		%r23(%r26)
9831012 	pdc,m		%r23(%r26)
9841013 	pdc,m		%r23(%r26)
985
-	cmpb,COND(<<)		%r26, %r25, 1b
1014
+	cmpb,COND(>>)	%r25, %r26, 1b /* predict taken */
9861015 	pdc,m		%r23(%r26)
9871016 
1017
+89:	ALTERNATIVE(88b, 89b, ALT_COND_NO_DCACHE, INSN_NOP)
9881018 	sync
9891019 	bv		%r0(%r2)
9901020 	nop
9911021 ENDPROC_CFI(purge_kernel_dcache_page_asm)
9921022 
9931023 ENTRY_CFI(flush_user_dcache_range_asm)
994
-	ldil		L%dcache_stride, %r1
1024
+88:	ldil		L%dcache_stride, %r1
9951025 	ldw		R%dcache_stride(%r1), %r23
9961026 	ldo		-1(%r23), %r21
9971027 	ANDCM		%r26, %r21, %r26
9981028 
999
-1:      cmpb,COND(<<),n	%r26, %r25, 1b
1029
+#ifdef CONFIG_64BIT
1030
+	depd,z		%r23, 59, 60, %r21
1031
+#else
1032
+	depw,z		%r23, 27, 28, %r21
1033
+#endif
1034
+	add		%r26, %r21, %r22
1035
+	cmpb,COND(>>),n	%r22, %r25, 2f /* predict not taken */
1036
+1:	add		%r22, %r21, %r22
1037
+	fdc,m		%r23(%sr3, %r26)
1038
+	fdc,m		%r23(%sr3, %r26)
1039
+	fdc,m		%r23(%sr3, %r26)
1040
+	fdc,m		%r23(%sr3, %r26)
1041
+	fdc,m		%r23(%sr3, %r26)
1042
+	fdc,m		%r23(%sr3, %r26)
1043
+	fdc,m		%r23(%sr3, %r26)
1044
+	fdc,m		%r23(%sr3, %r26)
1045
+	fdc,m		%r23(%sr3, %r26)
1046
+	fdc,m		%r23(%sr3, %r26)
1047
+	fdc,m		%r23(%sr3, %r26)
1048
+	fdc,m		%r23(%sr3, %r26)
1049
+	fdc,m		%r23(%sr3, %r26)
1050
+	fdc,m		%r23(%sr3, %r26)
1051
+	fdc,m		%r23(%sr3, %r26)
1052
+	cmpb,COND(<<=)	%r22, %r25, 1b /* predict taken */
10001053 	fdc,m		%r23(%sr3, %r26)
10011054 
1055
+2:	cmpb,COND(>>),n	%r25, %r26, 2b
1056
+	fdc,m		%r23(%sr3, %r26)
1057
+
1058
+89:	ALTERNATIVE(88b, 89b, ALT_COND_NO_DCACHE, INSN_NOP)
10021059 	sync
10031060 	bv		%r0(%r2)
10041061 	nop
10051062 ENDPROC_CFI(flush_user_dcache_range_asm)
10061063 
10071064 ENTRY_CFI(flush_kernel_dcache_range_asm)
1008
-	ldil		L%dcache_stride, %r1
1065
+88:	ldil		L%dcache_stride, %r1
10091066 	ldw		R%dcache_stride(%r1), %r23
10101067 	ldo		-1(%r23), %r21
10111068 	ANDCM		%r26, %r21, %r26
10121069 
1013
-1:      cmpb,COND(<<),n	%r26, %r25,1b
1070
+#ifdef CONFIG_64BIT
1071
+	depd,z		%r23, 59, 60, %r21
1072
+#else
1073
+	depw,z		%r23, 27, 28, %r21
1074
+#endif
1075
+	add		%r26, %r21, %r22
1076
+	cmpb,COND(>>),n	%r22, %r25, 2f /* predict not taken */
1077
+1:	add		%r22, %r21, %r22
1078
+	fdc,m		%r23(%r26)
1079
+	fdc,m		%r23(%r26)
1080
+	fdc,m		%r23(%r26)
1081
+	fdc,m		%r23(%r26)
1082
+	fdc,m		%r23(%r26)
1083
+	fdc,m		%r23(%r26)
1084
+	fdc,m		%r23(%r26)
1085
+	fdc,m		%r23(%r26)
1086
+	fdc,m		%r23(%r26)
1087
+	fdc,m		%r23(%r26)
1088
+	fdc,m		%r23(%r26)
1089
+	fdc,m		%r23(%r26)
1090
+	fdc,m		%r23(%r26)
1091
+	fdc,m		%r23(%r26)
1092
+	fdc,m		%r23(%r26)
1093
+	cmpb,COND(<<=)	%r22, %r25, 1b /* predict taken */
1094
+	fdc,m		%r23(%r26)
1095
+
1096
+2:	cmpb,COND(>>),n	%r25, %r26, 2b /* predict taken */
10141097 	fdc,m		%r23(%r26)
10151098 
10161099 	sync
1100
+89:	ALTERNATIVE(88b, 89b, ALT_COND_NO_DCACHE, INSN_NOP)
10171101 	syncdma
10181102 	bv		%r0(%r2)
10191103 	nop
10201104 ENDPROC_CFI(flush_kernel_dcache_range_asm)
10211105 
10221106 ENTRY_CFI(purge_kernel_dcache_range_asm)
1023
-	ldil		L%dcache_stride, %r1
1107
+88:	ldil		L%dcache_stride, %r1
10241108 	ldw		R%dcache_stride(%r1), %r23
10251109 	ldo		-1(%r23), %r21
10261110 	ANDCM		%r26, %r21, %r26
10271111 
1028
-1:      cmpb,COND(<<),n	%r26, %r25,1b
1112
+#ifdef CONFIG_64BIT
1113
+	depd,z		%r23, 59, 60, %r21
1114
+#else
1115
+	depw,z		%r23, 27, 28, %r21
1116
+#endif
1117
+	add		%r26, %r21, %r22
1118
+	cmpb,COND(>>),n	%r22, %r25, 2f /* predict not taken */
1119
+1:	add		%r22, %r21, %r22
1120
+	pdc,m		%r23(%r26)
1121
+	pdc,m		%r23(%r26)
1122
+	pdc,m		%r23(%r26)
1123
+	pdc,m		%r23(%r26)
1124
+	pdc,m		%r23(%r26)
1125
+	pdc,m		%r23(%r26)
1126
+	pdc,m		%r23(%r26)
1127
+	pdc,m		%r23(%r26)
1128
+	pdc,m		%r23(%r26)
1129
+	pdc,m		%r23(%r26)
1130
+	pdc,m		%r23(%r26)
1131
+	pdc,m		%r23(%r26)
1132
+	pdc,m		%r23(%r26)
1133
+	pdc,m		%r23(%r26)
1134
+	pdc,m		%r23(%r26)
1135
+	cmpb,COND(<<=)	%r22, %r25, 1b /* predict taken */
1136
+	pdc,m		%r23(%r26)
1137
+
1138
+2:	cmpb,COND(>>),n	%r25, %r26, 2b /* predict taken */
10291139 	pdc,m		%r23(%r26)
10301140 
10311141 	sync
1142
+89:	ALTERNATIVE(88b, 89b, ALT_COND_NO_DCACHE, INSN_NOP)
10321143 	syncdma
10331144 	bv		%r0(%r2)
10341145 	nop
10351146 ENDPROC_CFI(purge_kernel_dcache_range_asm)
10361147 
10371148 ENTRY_CFI(flush_user_icache_range_asm)
1038
-	ldil		L%icache_stride, %r1
1149
+88:	ldil		L%icache_stride, %r1
10391150 	ldw		R%icache_stride(%r1), %r23
10401151 	ldo		-1(%r23), %r21
10411152 	ANDCM		%r26, %r21, %r26
10421153 
1043
-1:      cmpb,COND(<<),n	%r26, %r25,1b
1154
+#ifdef CONFIG_64BIT
1155
+	depd,z		%r23, 59, 60, %r21
1156
+#else
1157
+	depw,z		%r23, 27, 28, %r21
1158
+#endif
1159
+	add		%r26, %r21, %r22
1160
+	cmpb,COND(>>),n	%r22, %r25, 2f /* predict not taken */
1161
+1:	add		%r22, %r21, %r22
1162
+	fic,m		%r23(%sr3, %r26)
1163
+	fic,m		%r23(%sr3, %r26)
1164
+	fic,m		%r23(%sr3, %r26)
1165
+	fic,m		%r23(%sr3, %r26)
1166
+	fic,m		%r23(%sr3, %r26)
1167
+	fic,m		%r23(%sr3, %r26)
1168
+	fic,m		%r23(%sr3, %r26)
1169
+	fic,m		%r23(%sr3, %r26)
1170
+	fic,m		%r23(%sr3, %r26)
1171
+	fic,m		%r23(%sr3, %r26)
1172
+	fic,m		%r23(%sr3, %r26)
1173
+	fic,m		%r23(%sr3, %r26)
1174
+	fic,m		%r23(%sr3, %r26)
1175
+	fic,m		%r23(%sr3, %r26)
1176
+	fic,m		%r23(%sr3, %r26)
1177
+	cmpb,COND(<<=)	%r22, %r25, 1b /* predict taken */
10441178 	fic,m		%r23(%sr3, %r26)
10451179 
1180
+2:	cmpb,COND(>>),n	%r25, %r26, 2b
1181
+	fic,m		%r23(%sr3, %r26)
1182
+
1183
+89:	ALTERNATIVE(88b, 89b, ALT_COND_NO_ICACHE, INSN_NOP)
10461184 	sync
10471185 	bv		%r0(%r2)
10481186 	nop
10491187 ENDPROC_CFI(flush_user_icache_range_asm)
10501188 
10511189 ENTRY_CFI(flush_kernel_icache_page)
1052
-	ldil		L%icache_stride, %r1
1190
+88:	ldil		L%icache_stride, %r1
10531191 	ldw		R%icache_stride(%r1), %r23
10541192 
10551193 #ifdef CONFIG_64BIT
....@@ -1076,23 +1214,51 @@
10761214 	fic,m		%r23(%sr4, %r26)
10771215 	fic,m		%r23(%sr4, %r26)
10781216 	fic,m		%r23(%sr4, %r26)
1079
-	cmpb,COND(<<)		%r26, %r25, 1b
1217
+	cmpb,COND(>>)	%r25, %r26, 1b /* predict taken */
10801218 	fic,m		%r23(%sr4, %r26)
10811219 
1220
+89:	ALTERNATIVE(88b, 89b, ALT_COND_NO_ICACHE, INSN_NOP)
10821221 	sync
10831222 	bv		%r0(%r2)
10841223 	nop
10851224 ENDPROC_CFI(flush_kernel_icache_page)
10861225 
10871226 ENTRY_CFI(flush_kernel_icache_range_asm)
1088
-	ldil		L%icache_stride, %r1
1227
+88:	ldil		L%icache_stride, %r1
10891228 	ldw		R%icache_stride(%r1), %r23
10901229 	ldo		-1(%r23), %r21
10911230 	ANDCM		%r26, %r21, %r26
10921231 
1093
-1:      cmpb,COND(<<),n	%r26, %r25, 1b
1232
+#ifdef CONFIG_64BIT
1233
+	depd,z		%r23, 59, 60, %r21
1234
+#else
1235
+	depw,z		%r23, 27, 28, %r21
1236
+#endif
1237
+	add		%r26, %r21, %r22
1238
+	cmpb,COND(>>),n	%r22, %r25, 2f /* predict not taken */
1239
+1:	add		%r22, %r21, %r22
1240
+	fic,m		%r23(%sr4, %r26)
1241
+	fic,m		%r23(%sr4, %r26)
1242
+	fic,m		%r23(%sr4, %r26)
1243
+	fic,m		%r23(%sr4, %r26)
1244
+	fic,m		%r23(%sr4, %r26)
1245
+	fic,m		%r23(%sr4, %r26)
1246
+	fic,m		%r23(%sr4, %r26)
1247
+	fic,m		%r23(%sr4, %r26)
1248
+	fic,m		%r23(%sr4, %r26)
1249
+	fic,m		%r23(%sr4, %r26)
1250
+	fic,m		%r23(%sr4, %r26)
1251
+	fic,m		%r23(%sr4, %r26)
1252
+	fic,m		%r23(%sr4, %r26)
1253
+	fic,m		%r23(%sr4, %r26)
1254
+	fic,m		%r23(%sr4, %r26)
1255
+	cmpb,COND(<<=)	%r22, %r25, 1b /* predict taken */
10941256 	fic,m		%r23(%sr4, %r26)
10951257 
1258
+2:	cmpb,COND(>>),n	%r25, %r26, 2b /* predict taken */
1259
+	fic,m		%r23(%sr4, %r26)
1260
+
1261
+89:	ALTERNATIVE(88b, 89b, ALT_COND_NO_ICACHE, INSN_NOP)
10961262 	sync
10971263 	bv		%r0(%r2)
10981264 	nop