blame | history | patch | commit | commitdiff
hc
2023-12-08 01573e231f18eb2d99162747186f59511f56b64d 
 kernel/arch/arm64/include/asm/alternative.h
....@@ -2,19 +2,13 @@
22 #ifndef __ASM_ALTERNATIVE_H
33 #define __ASM_ALTERNATIVE_H
44 
5
-#include <asm/cpucaps.h>
6
-#include <asm/insn.h>
7
-
8
-#define ARM64_CB_PATCH ARM64_NCAPS
5
+#include <asm/alternative-macros.h>
96 
107 #ifndef __ASSEMBLY__
118 
129 #include <linux/init.h>
1310 #include <linux/types.h>
1411 #include <linux/stddef.h>
15
-#include <linux/stringify.h>
16
-
17
-extern int alternatives_applied;
1812 
1913 struct alt_instr {
2014 	s32 orig_offset;	/* offset to original instruction */
....@@ -27,7 +21,9 @@
2721 typedef void (*alternative_cb_t)(struct alt_instr *alt,
2822 				 __le32 *origptr, __le32 *updptr, int nr_inst);
2923 
24
+void __init apply_boot_alternatives(void);
3025 void __init apply_alternatives_all(void);
26
+bool alternative_is_applied(u16 cpufeature);
3127 
3228 #ifdef CONFIG_MODULES
3329 void apply_alternatives_module(void *start, size_t length);
....@@ -35,264 +31,5 @@
3531 static inline void apply_alternatives_module(void *start, size_t length) { }
3632 #endif
3733 
38
-#define ALTINSTR_ENTRY(feature)					              \
39
-	" .word 661b - .\n"				/* label           */ \
40
-	" .word 663f - .\n"				/* new instruction */ \
41
-	" .hword " __stringify(feature) "\n"		/* feature bit     */ \
42
-	" .byte 662b-661b\n"				/* source len      */ \
43
-	" .byte 664f-663f\n"				/* replacement len */
44
-
45
-#define ALTINSTR_ENTRY_CB(feature, cb)					      \
46
-	" .word 661b - .\n"				/* label           */ \
47
-	" .word " __stringify(cb) "- .\n"		/* callback */	      \
48
-	" .hword " __stringify(feature) "\n"		/* feature bit     */ \
49
-	" .byte 662b-661b\n"				/* source len      */ \
50
-	" .byte 664f-663f\n"				/* replacement len */
51
-
52
-/*
53
- * alternative assembly primitive:
54
- *
55
- * If any of these .org directive fail, it means that insn1 and insn2
56
- * don't have the same length. This used to be written as
57
- *
58
- * .if ((664b-663b) != (662b-661b))
59
- * 	.error "Alternatives instruction length mismatch"
60
- * .endif
61
- *
62
- * but most assemblers die if insn1 or insn2 have a .inst. This should
63
- * be fixed in a binutils release posterior to 2.25.51.0.2 (anything
64
- * containing commit 4e4d08cf7399b606 or c1baaddf8861).
65
- *
66
- * Alternatives with callbacks do not generate replacement instructions.
67
- */
68
-#define __ALTERNATIVE_CFG(oldinstr, newinstr, feature, cfg_enabled)	\
69
-	".if "__stringify(cfg_enabled)" == 1\n"				\
70
-	"661:\n\t"							\
71
-	oldinstr "\n"							\
72
-	"662:\n"							\
73
-	".pushsection .altinstructions,\"a\"\n"				\
74
-	ALTINSTR_ENTRY(feature)						\
75
-	".popsection\n"							\
76
-	".pushsection .altinstr_replacement, \"a\"\n"			\
77
-	"663:\n\t"							\
78
-	newinstr "\n"							\
79
-	"664:\n\t"							\
80
-	".popsection\n\t"						\
81
-	".org	. - (664b-663b) + (662b-661b)\n\t"			\
82
-	".org	. - (662b-661b) + (664b-663b)\n"			\
83
-	".endif\n"
84
-
85
-#define __ALTERNATIVE_CFG_CB(oldinstr, feature, cfg_enabled, cb)	\
86
-	".if "__stringify(cfg_enabled)" == 1\n"				\
87
-	"661:\n\t"							\
88
-	oldinstr "\n"							\
89
-	"662:\n"							\
90
-	".pushsection .altinstructions,\"a\"\n"				\
91
-	ALTINSTR_ENTRY_CB(feature, cb)					\
92
-	".popsection\n"							\
93
-	"663:\n\t"							\
94
-	"664:\n\t"							\
95
-	".endif\n"
96
-
97
-#define _ALTERNATIVE_CFG(oldinstr, newinstr, feature, cfg, ...)	\
98
-	__ALTERNATIVE_CFG(oldinstr, newinstr, feature, IS_ENABLED(cfg))
99
-
100
-#define ALTERNATIVE_CB(oldinstr, cb) \
101
-	__ALTERNATIVE_CFG_CB(oldinstr, ARM64_CB_PATCH, 1, cb)
102
-#else
103
-
104
-#include <asm/assembler.h>
105
-
106
-.macro altinstruction_entry orig_offset alt_offset feature orig_len alt_len
107
-	.word \orig_offset - .
108
-	.word \alt_offset - .
109
-	.hword \feature
110
-	.byte \orig_len
111
-	.byte \alt_len
112
-.endm
113
-
114
-.macro alternative_insn insn1, insn2, cap, enable = 1
115
-	.if \enable
116
-661:	\insn1
117
-662:	.pushsection .altinstructions, "a"
118
-	altinstruction_entry 661b, 663f, \cap, 662b-661b, 664f-663f
119
-	.popsection
120
-	.pushsection .altinstr_replacement, "ax"
121
-663:	\insn2
122
-664:	.org	. - (664b-663b) + (662b-661b)
123
-	.org	. - (662b-661b) + (664b-663b)
124
-	.popsection
125
-	.endif
126
-.endm
127
-
128
-/*
129
- * Alternative sequences
130
- *
131
- * The code for the case where the capability is not present will be
132
- * assembled and linked as normal. There are no restrictions on this
133
- * code.
134
- *
135
- * The code for the case where the capability is present will be
136
- * assembled into a special section to be used for dynamic patching.
137
- * Code for that case must:
138
- *
139
- * 1. Be exactly the same length (in bytes) as the default code
140
- *    sequence.
141
- *
142
- * 2. Not contain a branch target that is used outside of the
143
- *    alternative sequence it is defined in (branches into an
144
- *    alternative sequence are not fixed up).
145
- */
146
-
147
-/*
148
- * Begin an alternative code sequence.
149
- */
150
-.macro alternative_if_not cap
151
-	.set .Lasm_alt_mode, 0
152
-	.pushsection .altinstructions, "a"
153
-	altinstruction_entry 661f, 663f, \cap, 662f-661f, 664f-663f
154
-	.popsection
155
-661:
156
-.endm
157
-
158
-.macro alternative_if cap
159
-	.set .Lasm_alt_mode, 1
160
-	.pushsection .altinstructions, "a"
161
-	altinstruction_entry 663f, 661f, \cap, 664f-663f, 662f-661f
162
-	.popsection
163
-	.pushsection .altinstr_replacement, "ax"
164
-	.align 2	/* So GAS knows label 661 is suitably aligned */
165
-661:
166
-.endm
167
-
168
-.macro alternative_cb cb
169
-	.set .Lasm_alt_mode, 0
170
-	.pushsection .altinstructions, "a"
171
-	altinstruction_entry 661f, \cb, ARM64_CB_PATCH, 662f-661f, 0
172
-	.popsection
173
-661:
174
-.endm
175
-
176
-/*
177
- * Provide the other half of the alternative code sequence.
178
- */
179
-.macro alternative_else
180
-662:
181
-	.if .Lasm_alt_mode==0
182
-	.pushsection .altinstr_replacement, "ax"
183
-	.else
184
-	.popsection
185
-	.endif
186
-663:
187
-.endm
188
-
189
-/*
190
- * Complete an alternative code sequence.
191
- */
192
-.macro alternative_endif
193
-664:
194
-	.org	. - (664b-663b) + (662b-661b)
195
-	.org	. - (662b-661b) + (664b-663b)
196
-	.if .Lasm_alt_mode==0
197
-	.popsection
198
-	.endif
199
-.endm
200
-
201
-/*
202
- * Callback-based alternative epilogue
203
- */
204
-.macro alternative_cb_end
205
-662:
206
-.endm
207
-
208
-/*
209
- * Provides a trivial alternative or default sequence consisting solely
210
- * of NOPs. The number of NOPs is chosen automatically to match the
211
- * previous case.
212
- */
213
-.macro alternative_else_nop_endif
214
-alternative_else
215
-	nops	(662b-661b) / AARCH64_INSN_SIZE
216
-alternative_endif
217
-.endm
218
-
219
-#define _ALTERNATIVE_CFG(insn1, insn2, cap, cfg, ...)	\
220
-	alternative_insn insn1, insn2, cap, IS_ENABLED(cfg)
221
-
222
-.macro user_alt, label, oldinstr, newinstr, cond
223
-9999:	alternative_insn "\oldinstr", "\newinstr", \cond
224
-	_asm_extable 9999b, \label
225
-.endm
226
-
227
-/*
228
- * Generate the assembly for UAO alternatives with exception table entries.
229
- * This is complicated as there is no post-increment or pair versions of the
230
- * unprivileged instructions, and USER() only works for single instructions.
231
- */
232
-#ifdef CONFIG_ARM64_UAO
233
-	.macro uao_ldp l, reg1, reg2, addr, post_inc
234
-		alternative_if_not ARM64_HAS_UAO
235
-8888:			ldp	\reg1, \reg2, [\addr], \post_inc;
236
-8889:			nop;
237
-			nop;
238
-		alternative_else
239
-			ldtr	\reg1, [\addr];
240
-			ldtr	\reg2, [\addr, #8];
241
-			add	\addr, \addr, \post_inc;
242
-		alternative_endif
243
-
244
-		_asm_extable	8888b,\l;
245
-		_asm_extable	8889b,\l;
246
-	.endm
247
-
248
-	.macro uao_stp l, reg1, reg2, addr, post_inc
249
-		alternative_if_not ARM64_HAS_UAO
250
-8888:			stp	\reg1, \reg2, [\addr], \post_inc;
251
-8889:			nop;
252
-			nop;
253
-		alternative_else
254
-			sttr	\reg1, [\addr];
255
-			sttr	\reg2, [\addr, #8];
256
-			add	\addr, \addr, \post_inc;
257
-		alternative_endif
258
-
259
-		_asm_extable	8888b,\l;
260
-		_asm_extable	8889b,\l;
261
-	.endm
262
-
263
-	.macro uao_user_alternative l, inst, alt_inst, reg, addr, post_inc
264
-		alternative_if_not ARM64_HAS_UAO
265
-8888:			\inst	\reg, [\addr], \post_inc;
266
-			nop;
267
-		alternative_else
268
-			\alt_inst	\reg, [\addr];
269
-			add		\addr, \addr, \post_inc;
270
-		alternative_endif
271
-
272
-		_asm_extable	8888b,\l;
273
-	.endm
274
-#else
275
-	.macro uao_ldp l, reg1, reg2, addr, post_inc
276
-		USER(\l, ldp \reg1, \reg2, [\addr], \post_inc)
277
-	.endm
278
-	.macro uao_stp l, reg1, reg2, addr, post_inc
279
-		USER(\l, stp \reg1, \reg2, [\addr], \post_inc)
280
-	.endm
281
-	.macro uao_user_alternative l, inst, alt_inst, reg, addr, post_inc
282
-		USER(\l, \inst \reg, [\addr], \post_inc)
283
-	.endm
284
-#endif
285
-
286
-#endif  /*  __ASSEMBLY__  */
287
-
288
-/*
289
- * Usage: asm(ALTERNATIVE(oldinstr, newinstr, feature));
290
- *
291
- * Usage: asm(ALTERNATIVE(oldinstr, newinstr, feature, CONFIG_FOO));
292
- * N.B. If CONFIG_FOO is specified, but not selected, the whole block
293
- *      will be omitted, including oldinstr.
294
- */
295
-#define ALTERNATIVE(oldinstr, newinstr, ...)   \
296
-	_ALTERNATIVE_CFG(oldinstr, newinstr, __VA_ARGS__, 1)
297
-
34
+#endif /* __ASSEMBLY__ */
29835 #endif /* __ASM_ALTERNATIVE_H */