forked from ~ljy/RK356X_SDK_RELEASE

hc
2024-05-10 37f49e37ab4cb5d0bc4c60eb5c6d4dd57db767bb
kernel/arch/x86/kernel/jump_label.c
....@@ -16,15 +16,7 @@
1616 #include <asm/alternative.h>
1717 #include <asm/text-patching.h>
1818
19
-union jump_code_union {
20
- char code[JUMP_LABEL_NOP_SIZE];
21
- struct {
22
- char jump;
23
- int offset;
24
- } __attribute__((packed));
25
-};
26
-
27
-static void bug_at(unsigned char *ip, int line)
19
+static void bug_at(const void *ip, int line)
2820 {
2921 /*
3022 * The location is not an op that we were expecting.
....@@ -35,80 +27,103 @@
3527 BUG();
3628 }
3729
38
-static void __ref __jump_label_transform(struct jump_entry *entry,
39
- enum jump_label_type type,
40
- void *(*poker)(void *, const void *, size_t),
41
- int init)
30
+static const void *
31
+__jump_label_set_jump_code(struct jump_entry *entry, enum jump_label_type type, int init)
4232 {
43
- union jump_code_union code;
4433 const unsigned char default_nop[] = { STATIC_KEY_INIT_NOP };
4534 const unsigned char *ideal_nop = ideal_nops[NOP_ATOMIC5];
35
+ const void *expect, *code;
36
+ const void *addr, *dest;
37
+ int line;
4638
47
- if (early_boot_irqs_disabled)
48
- poker = text_poke_early;
39
+ addr = (void *)jump_entry_code(entry);
40
+ dest = (void *)jump_entry_target(entry);
4941
50
- if (type == JUMP_LABEL_JMP) {
51
- if (init) {
52
- /*
53
- * Jump label is enabled for the first time.
54
- * So we expect a default_nop...
55
- */
56
- if (unlikely(memcmp((void *)entry->code, default_nop, 5)
57
- != 0))
58
- bug_at((void *)entry->code, __LINE__);
59
- } else {
60
- /*
61
- * ...otherwise expect an ideal_nop. Otherwise
62
- * something went horribly wrong.
63
- */
64
- if (unlikely(memcmp((void *)entry->code, ideal_nop, 5)
65
- != 0))
66
- bug_at((void *)entry->code, __LINE__);
67
- }
42
+ code = text_gen_insn(JMP32_INSN_OPCODE, addr, dest);
6843
69
- code.jump = 0xe9;
70
- code.offset = entry->target -
71
- (entry->code + JUMP_LABEL_NOP_SIZE);
44
+ if (init) {
45
+ expect = default_nop; line = __LINE__;
46
+ } else if (type == JUMP_LABEL_JMP) {
47
+ expect = ideal_nop; line = __LINE__;
7248 } else {
73
- /*
74
- * We are disabling this jump label. If it is not what
75
- * we think it is, then something must have gone wrong.
76
- * If this is the first initialization call, then we
77
- * are converting the default nop to the ideal nop.
78
- */
79
- if (init) {
80
- if (unlikely(memcmp((void *)entry->code, default_nop, 5) != 0))
81
- bug_at((void *)entry->code, __LINE__);
82
- } else {
83
- code.jump = 0xe9;
84
- code.offset = entry->target -
85
- (entry->code + JUMP_LABEL_NOP_SIZE);
86
- if (unlikely(memcmp((void *)entry->code, &code, 5) != 0))
87
- bug_at((void *)entry->code, __LINE__);
88
- }
89
- memcpy(&code, ideal_nops[NOP_ATOMIC5], JUMP_LABEL_NOP_SIZE);
49
+ expect = code; line = __LINE__;
9050 }
9151
52
+ if (memcmp(addr, expect, JUMP_LABEL_NOP_SIZE))
53
+ bug_at(addr, line);
54
+
55
+ if (type == JUMP_LABEL_NOP)
56
+ code = ideal_nop;
57
+
58
+ return code;
59
+}
60
+
61
+static inline void __jump_label_transform(struct jump_entry *entry,
62
+ enum jump_label_type type,
63
+ int init)
64
+{
65
+ const void *opcode = __jump_label_set_jump_code(entry, type, init);
66
+
9267 /*
93
- * Make text_poke_bp() a default fallback poker.
68
+ * As long as only a single processor is running and the code is still
69
+ * not marked as RO, text_poke_early() can be used; Checking that
70
+ * system_state is SYSTEM_BOOTING guarantees it. It will be set to
71
+ * SYSTEM_SCHEDULING before other cores are awaken and before the
72
+ * code is write-protected.
9473 *
9574 * At the time the change is being done, just ignore whether we
9675 * are doing nop -> jump or jump -> nop transition, and assume
9776 * always nop being the 'currently valid' instruction
98
- *
9977 */
100
- if (poker)
101
- (*poker)((void *)entry->code, &code, JUMP_LABEL_NOP_SIZE);
102
- else
103
- text_poke_bp((void *)entry->code, &code, JUMP_LABEL_NOP_SIZE,
104
- (void *)entry->code + JUMP_LABEL_NOP_SIZE);
78
+ if (init || system_state == SYSTEM_BOOTING) {
79
+ text_poke_early((void *)jump_entry_code(entry), opcode,
80
+ JUMP_LABEL_NOP_SIZE);
81
+ return;
82
+ }
83
+
84
+ text_poke_bp((void *)jump_entry_code(entry), opcode, JUMP_LABEL_NOP_SIZE, NULL);
85
+}
86
+
87
+static void __ref jump_label_transform(struct jump_entry *entry,
88
+ enum jump_label_type type,
89
+ int init)
90
+{
91
+ mutex_lock(&text_mutex);
92
+ __jump_label_transform(entry, type, init);
93
+ mutex_unlock(&text_mutex);
10594 }
10695
10796 void arch_jump_label_transform(struct jump_entry *entry,
10897 enum jump_label_type type)
10998 {
99
+ jump_label_transform(entry, type, 0);
100
+}
101
+
102
+bool arch_jump_label_transform_queue(struct jump_entry *entry,
103
+ enum jump_label_type type)
104
+{
105
+ const void *opcode;
106
+
107
+ if (system_state == SYSTEM_BOOTING) {
108
+ /*
109
+ * Fallback to the non-batching mode.
110
+ */
111
+ arch_jump_label_transform(entry, type);
112
+ return true;
113
+ }
114
+
110115 mutex_lock(&text_mutex);
111
- __jump_label_transform(entry, type, NULL, 0);
116
+ opcode = __jump_label_set_jump_code(entry, type, 0);
117
+ text_poke_queue((void *)jump_entry_code(entry),
118
+ opcode, JUMP_LABEL_NOP_SIZE, NULL);
119
+ mutex_unlock(&text_mutex);
120
+ return true;
121
+}
122
+
123
+void arch_jump_label_transform_apply(void)
124
+{
125
+ mutex_lock(&text_mutex);
126
+ text_poke_finish();
112127 mutex_unlock(&text_mutex);
113128 }
114129
....@@ -138,5 +153,5 @@
138153 jlstate = JL_STATE_NO_UPDATE;
139154 }
140155 if (jlstate == JL_STATE_UPDATE)
141
- __jump_label_transform(entry, type, text_poke_early, 1);
156
+ jump_label_transform(entry, type, 1);
142157 }