forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2023-12-11 d2ccde1c8e90d38cee87a1b0309ad2827f3fd30d 
 kernel/arch/x86/include/asm/alternative.h
....@@ -2,12 +2,16 @@
22 #ifndef _ASM_X86_ALTERNATIVE_H
33 #define _ASM_X86_ALTERNATIVE_H
44 
5
-#ifndef __ASSEMBLY__
6
-
75 #include <linux/types.h>
8
-#include <linux/stddef.h>
96 #include <linux/stringify.h>
107 #include <asm/asm.h>
8
+
9
+#define ALTINSTR_FLAG_INV	(1 << 15)
10
+#define ALT_NOT(feat)		((feat) | ALTINSTR_FLAG_INV)
11
+
12
+#ifndef __ASSEMBLY__
13
+
14
+#include <linux/stddef.h>
1115 
1216 /*
1317  * Alternative inline assembly for SMP.
....@@ -45,13 +49,22 @@
4549 #define LOCK_PREFIX ""
4650 #endif
4751 
52
+/*
53
+ * objtool annotation to ignore the alternatives and only consider the original
54
+ * instruction(s).
55
+ */
56
+#define ANNOTATE_IGNORE_ALTERNATIVE				\
57
+	"999:\n\t"						\
58
+	".pushsection .discard.ignore_alts\n\t"			\
59
+	".long 999b - .\n\t"					\
60
+	".popsection\n\t"
61
+
4862 struct alt_instr {
4963 	s32 instr_offset;	/* original instruction */
5064 	s32 repl_offset;	/* offset to replacement instruction */
5165 	u16 cpuid;		/* cpuid bit set for replacement */
5266 	u8  instrlen;		/* length of original instruction */
5367 	u8  replacementlen;	/* length of new instruction */
54
-	u8  padlen;		/* length of build-time padding */
5568 } __packed;
5669 
5770 /*
....@@ -62,6 +75,8 @@
6275 
6376 extern void alternative_instructions(void);
6477 extern void apply_alternatives(struct alt_instr *start, struct alt_instr *end);
78
+extern void apply_retpolines(s32 *start, s32 *end);
79
+extern void apply_returns(s32 *start, s32 *end);
6580 
6681 struct module;
6782 
....@@ -90,17 +105,15 @@
90105 
91106 #define alt_end_marker		"663"
92107 #define alt_slen		"662b-661b"
93
-#define alt_pad_len		alt_end_marker"b-662b"
94108 #define alt_total_slen		alt_end_marker"b-661b"
95109 #define alt_rlen(num)		e_replacement(num)"f-"b_replacement(num)"f"
96110 
97
-#define __OLDINSTR(oldinstr, num)					\
98
-	"661:\n\t" oldinstr "\n662:\n"					\
99
-	".skip -(((" alt_rlen(num) ")-(" alt_slen ")) > 0) * "		\
100
-		"((" alt_rlen(num) ")-(" alt_slen ")),0x90\n"
101
-
102111 #define OLDINSTR(oldinstr, num)						\
103
-	__OLDINSTR(oldinstr, num)					\
112
+	"# ALT: oldnstr\n"						\
113
+	"661:\n\t" oldinstr "\n662:\n"					\
114
+	"# ALT: padding\n"						\
115
+	".skip -(((" alt_rlen(num) ")-(" alt_slen ")) > 0) * "		\
116
+		"((" alt_rlen(num) ")-(" alt_slen ")),0x90\n"		\
104117 	alt_end_marker ":\n"
105118 
106119 /*
....@@ -116,9 +129,21 @@
116129  * additionally longer than the first replacement alternative.
117130  */
118131 #define OLDINSTR_2(oldinstr, num1, num2) \
132
+	"# ALT: oldinstr2\n"									\
119133 	"661:\n\t" oldinstr "\n662:\n"								\
134
+	"# ALT: padding2\n"									\
120135 	".skip -((" alt_max_short(alt_rlen(num1), alt_rlen(num2)) " - (" alt_slen ")) > 0) * "	\
121136 		"(" alt_max_short(alt_rlen(num1), alt_rlen(num2)) " - (" alt_slen ")), 0x90\n"	\
137
+	alt_end_marker ":\n"
138
+
139
+#define OLDINSTR_3(oldinsn, n1, n2, n3)								\
140
+	"# ALT: oldinstr3\n"									\
141
+	"661:\n\t" oldinsn "\n662:\n"								\
142
+	"# ALT: padding3\n"									\
143
+	".skip -((" alt_max_short(alt_max_short(alt_rlen(n1), alt_rlen(n2)), alt_rlen(n3))	\
144
+		" - (" alt_slen ")) > 0) * "							\
145
+		"(" alt_max_short(alt_max_short(alt_rlen(n1), alt_rlen(n2)), alt_rlen(n3))	\
146
+		" - (" alt_slen ")), 0x90\n"							\
122147 	alt_end_marker ":\n"
123148 
124149 #define ALTINSTR_ENTRY(feature, num)					      \
....@@ -126,11 +151,11 @@
126151 	" .long " b_replacement(num)"f - .\n"		/* new instruction */ \
127152 	" .word " __stringify(feature) "\n"		/* feature bit     */ \
128153 	" .byte " alt_total_slen "\n"			/* source len      */ \
129
-	" .byte " alt_rlen(num) "\n"			/* replacement len */ \
130
-	" .byte " alt_pad_len "\n"			/* pad len */
154
+	" .byte " alt_rlen(num) "\n"			/* replacement len */
131155 
132
-#define ALTINSTR_REPLACEMENT(newinstr, feature, num)	/* replacement */     \
133
-	b_replacement(num)":\n\t" newinstr "\n" e_replacement(num) ":\n\t"
156
+#define ALTINSTR_REPLACEMENT(newinstr, feature, num)	/* replacement */	\
157
+	"# ALT: replacement " #num "\n"						\
158
+	b_replacement(num)":\n\t" newinstr "\n" e_replacement(num) ":\n"
134159 
135160 /* alternative assembly primitive: */
136161 #define ALTERNATIVE(oldinstr, newinstr, feature)			\
....@@ -153,6 +178,24 @@
153178 	ALTINSTR_REPLACEMENT(newinstr2, feature2, 2)			\
154179 	".popsection\n"
155180 
181
+/* If @feature is set, patch in @newinstr_yes, otherwise @newinstr_no. */
182
+#define ALTERNATIVE_TERNARY(oldinstr, feature, newinstr_yes, newinstr_no) \
183
+	ALTERNATIVE_2(oldinstr, newinstr_no, X86_FEATURE_ALWAYS,	\
184
+		      newinstr_yes, feature)
185
+
186
+#define ALTERNATIVE_3(oldinsn, newinsn1, feat1, newinsn2, feat2, newinsn3, feat3) \
187
+	OLDINSTR_3(oldinsn, 1, 2, 3)						\
188
+	".pushsection .altinstructions,\"a\"\n"					\
189
+	ALTINSTR_ENTRY(feat1, 1)						\
190
+	ALTINSTR_ENTRY(feat2, 2)						\
191
+	ALTINSTR_ENTRY(feat3, 3)						\
192
+	".popsection\n"								\
193
+	".pushsection .altinstr_replacement, \"ax\"\n"				\
194
+	ALTINSTR_REPLACEMENT(newinsn1, feat1, 1)				\
195
+	ALTINSTR_REPLACEMENT(newinsn2, feat2, 2)				\
196
+	ALTINSTR_REPLACEMENT(newinsn3, feat3, 3)				\
197
+	".popsection\n"
198
+
156199 /*
157200  * Alternative instructions for different CPU types or capabilities.
158201  *
....@@ -166,24 +209,24 @@
166209  * without volatile and memory clobber.
167210  */
168211 #define alternative(oldinstr, newinstr, feature)			\
169
-	asm volatile (ALTERNATIVE(oldinstr, newinstr, feature) : : : "memory")
212
+	asm_inline volatile (ALTERNATIVE(oldinstr, newinstr, feature) : : : "memory")
170213 
171214 #define alternative_2(oldinstr, newinstr1, feature1, newinstr2, feature2) \
172
-	asm volatile(ALTERNATIVE_2(oldinstr, newinstr1, feature1, newinstr2, feature2) ::: "memory")
215
+	asm_inline volatile(ALTERNATIVE_2(oldinstr, newinstr1, feature1, newinstr2, feature2) ::: "memory")
216
+
217
+#define alternative_ternary(oldinstr, feature, newinstr_yes, newinstr_no) \
218
+	asm_inline volatile(ALTERNATIVE_TERNARY(oldinstr, feature, newinstr_yes, newinstr_no) ::: "memory")
173219 
174220 /*
175221  * Alternative inline assembly with input.
176222  *
177
- * Pecularities:
223
+ * Peculiarities:
178224  * No memory clobber here.
179225  * Argument numbers start with 1.
180
- * Best is to use constraints that are fixed size (like (%1) ... "r")
181
- * If you use variable sized constraints like "m" or "g" in the
182
- * replacement make sure to pad to the worst case length.
183226  * Leaving an unused argument 0 to keep API compatibility.
184227  */
185228 #define alternative_input(oldinstr, newinstr, feature, input...)	\
186
-	asm volatile (ALTERNATIVE(oldinstr, newinstr, feature)		\
229
+	asm_inline volatile (ALTERNATIVE(oldinstr, newinstr, feature)	\
187230 		: : "i" (0), ## input)
188231 
189232 /*
....@@ -196,18 +239,18 @@
196239  */
197240 #define alternative_input_2(oldinstr, newinstr1, feature1, newinstr2,	     \
198241 			   feature2, input...)				     \
199
-	asm volatile(ALTERNATIVE_2(oldinstr, newinstr1, feature1,	     \
242
+	asm_inline volatile(ALTERNATIVE_2(oldinstr, newinstr1, feature1,     \
200243 		newinstr2, feature2)					     \
201244 		: : "i" (0), ## input)
202245 
203246 /* Like alternative_input, but with a single output argument */
204247 #define alternative_io(oldinstr, newinstr, feature, output, input...)	\
205
-	asm volatile (ALTERNATIVE(oldinstr, newinstr, feature)		\
248
+	asm_inline volatile (ALTERNATIVE(oldinstr, newinstr, feature)	\
206249 		: output : "i" (0), ## input)
207250 
208251 /* Like alternative_io, but for replacing a direct call with another one. */
209252 #define alternative_call(oldfunc, newfunc, feature, output, input...)	\
210
-	asm volatile (ALTERNATIVE("call %P[old]", "call %P[new]", feature) \
253
+	asm_inline volatile (ALTERNATIVE("call %P[old]", "call %P[new]", feature) \
211254 		: output : [old] "i" (oldfunc), [new] "i" (newfunc), ## input)
212255 
213256 /*
....@@ -218,7 +261,7 @@
218261  */
219262 #define alternative_call_2(oldfunc, newfunc1, feature1, newfunc2, feature2,   \
220263 			   output, input...)				      \
221
-	asm volatile (ALTERNATIVE_2("call %P[old]", "call %P[new1]", feature1,\
264
+	asm_inline volatile (ALTERNATIVE_2("call %P[old]", "call %P[new1]", feature1,\
222265 		"call %P[new2]", feature2)				      \
223266 		: output, ASM_CALL_CONSTRAINT				      \
224267 		: [old] "i" (oldfunc), [new1] "i" (newfunc1),		      \
....@@ -236,6 +279,115 @@
236279  */
237280 #define ASM_NO_INPUT_CLOBBER(clbr...) "i" (0) : clbr
238281 
282
+#else /* __ASSEMBLY__ */
283
+
284
+#ifdef CONFIG_SMP
285
+	.macro LOCK_PREFIX
286
+672:	lock
287
+	.pushsection .smp_locks,"a"
288
+	.balign 4
289
+	.long 672b - .
290
+	.popsection
291
+	.endm
292
+#else
293
+	.macro LOCK_PREFIX
294
+	.endm
295
+#endif
296
+
297
+/*
298
+ * objtool annotation to ignore the alternatives and only consider the original
299
+ * instruction(s).
300
+ */
301
+.macro ANNOTATE_IGNORE_ALTERNATIVE
302
+	.Lannotate_\@:
303
+	.pushsection .discard.ignore_alts
304
+	.long .Lannotate_\@ - .
305
+	.popsection
306
+.endm
307
+
308
+/*
309
+ * Issue one struct alt_instr descriptor entry (need to put it into
310
+ * the section .altinstructions, see below). This entry contains
311
+ * enough information for the alternatives patching code to patch an
312
+ * instruction. See apply_alternatives().
313
+ */
314
+.macro altinstruction_entry orig alt feature orig_len alt_len
315
+	.long \orig - .
316
+	.long \alt - .
317
+	.word \feature
318
+	.byte \orig_len
319
+	.byte \alt_len
320
+.endm
321
+
322
+/*
323
+ * Define an alternative between two instructions. If @feature is
324
+ * present, early code in apply_alternatives() replaces @oldinstr with
325
+ * @newinstr. ".skip" directive takes care of proper instruction padding
326
+ * in case @newinstr is longer than @oldinstr.
327
+ */
328
+.macro ALTERNATIVE oldinstr, newinstr, feature
329
+140:
330
+	\oldinstr
331
+141:
332
+	.skip -(((144f-143f)-(141b-140b)) > 0) * ((144f-143f)-(141b-140b)),0x90
333
+142:
334
+
335
+	.pushsection .altinstructions,"a"
336
+	altinstruction_entry 140b,143f,\feature,142b-140b,144f-143f
337
+	.popsection
338
+
339
+	.pushsection .altinstr_replacement,"ax"
340
+143:
341
+	\newinstr
342
+144:
343
+	.popsection
344
+.endm
345
+
346
+#define old_len			141b-140b
347
+#define new_len1		144f-143f
348
+#define new_len2		145f-144f
349
+
350
+/*
351
+ * gas compatible max based on the idea from:
352
+ * http://graphics.stanford.edu/~seander/bithacks.html#IntegerMinOrMax
353
+ *
354
+ * The additional "-" is needed because gas uses a "true" value of -1.
355
+ */
356
+#define alt_max_short(a, b)	((a) ^ (((a) ^ (b)) & -(-((a) < (b)))))
357
+
358
+
359
+/*
360
+ * Same as ALTERNATIVE macro above but for two alternatives. If CPU
361
+ * has @feature1, it replaces @oldinstr with @newinstr1. If CPU has
362
+ * @feature2, it replaces @oldinstr with @feature2.
363
+ */
364
+.macro ALTERNATIVE_2 oldinstr, newinstr1, feature1, newinstr2, feature2
365
+140:
366
+	\oldinstr
367
+141:
368
+	.skip -((alt_max_short(new_len1, new_len2) - (old_len)) > 0) * \
369
+		(alt_max_short(new_len1, new_len2) - (old_len)),0x90
370
+142:
371
+
372
+	.pushsection .altinstructions,"a"
373
+	altinstruction_entry 140b,143f,\feature1,142b-140b,144f-143f
374
+	altinstruction_entry 140b,144f,\feature2,142b-140b,145f-144f
375
+	.popsection
376
+
377
+	.pushsection .altinstr_replacement,"ax"
378
+143:
379
+	\newinstr1
380
+144:
381
+	\newinstr2
382
+145:
383
+	.popsection
384
+.endm
385
+
386
+/* If @feature is set, patch in @newinstr_yes, otherwise @newinstr_no. */
387
+#define ALTERNATIVE_TERNARY(oldinstr, feature, newinstr_yes, newinstr_no) \
388
+	ALTERNATIVE_2 oldinstr, newinstr_no, X86_FEATURE_ALWAYS,	\
389
+	newinstr_yes, feature
390
+
239391 #endif /* __ASSEMBLY__ */
240392 
241393 #endif /* _ASM_X86_ALTERNATIVE_H */