.. | .. |
---|
16 | 16 | #include <asm/alternative.h> |
---|
17 | 17 | #include <asm/text-patching.h> |
---|
18 | 18 | |
---|
19 | | -union jump_code_union { |
---|
20 | | - char code[JUMP_LABEL_NOP_SIZE]; |
---|
21 | | - struct { |
---|
22 | | - char jump; |
---|
23 | | - int offset; |
---|
24 | | - } __attribute__((packed)); |
---|
25 | | -}; |
---|
26 | | - |
---|
27 | | -static void bug_at(unsigned char *ip, int line) |
---|
| 19 | +static void bug_at(const void *ip, int line) |
---|
28 | 20 | { |
---|
29 | 21 | /* |
---|
30 | 22 | * The location is not an op that we were expecting. |
---|
.. | .. |
---|
35 | 27 | BUG(); |
---|
36 | 28 | } |
---|
37 | 29 | |
---|
38 | | -static void __ref __jump_label_transform(struct jump_entry *entry, |
---|
39 | | - enum jump_label_type type, |
---|
40 | | - void *(*poker)(void *, const void *, size_t), |
---|
41 | | - int init) |
---|
| 30 | +static const void * |
---|
| 31 | +__jump_label_set_jump_code(struct jump_entry *entry, enum jump_label_type type, int init) |
---|
42 | 32 | { |
---|
43 | | - union jump_code_union code; |
---|
44 | 33 | const unsigned char default_nop[] = { STATIC_KEY_INIT_NOP }; |
---|
45 | 34 | const unsigned char *ideal_nop = ideal_nops[NOP_ATOMIC5]; |
---|
| 35 | + const void *expect, *code; |
---|
| 36 | + const void *addr, *dest; |
---|
| 37 | + int line; |
---|
46 | 38 | |
---|
47 | | - if (early_boot_irqs_disabled) |
---|
48 | | - poker = text_poke_early; |
---|
| 39 | + addr = (void *)jump_entry_code(entry); |
---|
| 40 | + dest = (void *)jump_entry_target(entry); |
---|
49 | 41 | |
---|
50 | | - if (type == JUMP_LABEL_JMP) { |
---|
51 | | - if (init) { |
---|
52 | | - /* |
---|
53 | | - * Jump label is enabled for the first time. |
---|
54 | | - * So we expect a default_nop... |
---|
55 | | - */ |
---|
56 | | - if (unlikely(memcmp((void *)entry->code, default_nop, 5) |
---|
57 | | - != 0)) |
---|
58 | | - bug_at((void *)entry->code, __LINE__); |
---|
59 | | - } else { |
---|
60 | | - /* |
---|
61 | | - * ...otherwise expect an ideal_nop. Otherwise |
---|
62 | | - * something went horribly wrong. |
---|
63 | | - */ |
---|
64 | | - if (unlikely(memcmp((void *)entry->code, ideal_nop, 5) |
---|
65 | | - != 0)) |
---|
66 | | - bug_at((void *)entry->code, __LINE__); |
---|
67 | | - } |
---|
| 42 | + code = text_gen_insn(JMP32_INSN_OPCODE, addr, dest); |
---|
68 | 43 | |
---|
69 | | - code.jump = 0xe9; |
---|
70 | | - code.offset = entry->target - |
---|
71 | | - (entry->code + JUMP_LABEL_NOP_SIZE); |
---|
| 44 | + if (init) { |
---|
| 45 | + expect = default_nop; line = __LINE__; |
---|
| 46 | + } else if (type == JUMP_LABEL_JMP) { |
---|
| 47 | + expect = ideal_nop; line = __LINE__; |
---|
72 | 48 | } else { |
---|
73 | | - /* |
---|
74 | | - * We are disabling this jump label. If it is not what |
---|
75 | | - * we think it is, then something must have gone wrong. |
---|
76 | | - * If this is the first initialization call, then we |
---|
77 | | - * are converting the default nop to the ideal nop. |
---|
78 | | - */ |
---|
79 | | - if (init) { |
---|
80 | | - if (unlikely(memcmp((void *)entry->code, default_nop, 5) != 0)) |
---|
81 | | - bug_at((void *)entry->code, __LINE__); |
---|
82 | | - } else { |
---|
83 | | - code.jump = 0xe9; |
---|
84 | | - code.offset = entry->target - |
---|
85 | | - (entry->code + JUMP_LABEL_NOP_SIZE); |
---|
86 | | - if (unlikely(memcmp((void *)entry->code, &code, 5) != 0)) |
---|
87 | | - bug_at((void *)entry->code, __LINE__); |
---|
88 | | - } |
---|
89 | | - memcpy(&code, ideal_nops[NOP_ATOMIC5], JUMP_LABEL_NOP_SIZE); |
---|
| 49 | + expect = code; line = __LINE__; |
---|
90 | 50 | } |
---|
91 | 51 | |
---|
| 52 | + if (memcmp(addr, expect, JUMP_LABEL_NOP_SIZE)) |
---|
| 53 | + bug_at(addr, line); |
---|
| 54 | + |
---|
| 55 | + if (type == JUMP_LABEL_NOP) |
---|
| 56 | + code = ideal_nop; |
---|
| 57 | + |
---|
| 58 | + return code; |
---|
| 59 | +} |
---|
| 60 | + |
---|
| 61 | +static inline void __jump_label_transform(struct jump_entry *entry, |
---|
| 62 | + enum jump_label_type type, |
---|
| 63 | + int init) |
---|
| 64 | +{ |
---|
| 65 | + const void *opcode = __jump_label_set_jump_code(entry, type, init); |
---|
| 66 | + |
---|
92 | 67 | /* |
---|
93 | | - * Make text_poke_bp() a default fallback poker. |
---|
| 68 | + * As long as only a single processor is running and the code is still |
---|
| 69 | + * not marked as RO, text_poke_early() can be used; Checking that |
---|
| 70 | + * system_state is SYSTEM_BOOTING guarantees it. It will be set to |
---|
| 71 | + * SYSTEM_SCHEDULING before other cores are awaken and before the |
---|
| 72 | + * code is write-protected. |
---|
94 | 73 | * |
---|
95 | 74 | * At the time the change is being done, just ignore whether we |
---|
96 | 75 | * are doing nop -> jump or jump -> nop transition, and assume |
---|
97 | 76 | * always nop being the 'currently valid' instruction |
---|
98 | | - * |
---|
99 | 77 | */ |
---|
100 | | - if (poker) |
---|
101 | | - (*poker)((void *)entry->code, &code, JUMP_LABEL_NOP_SIZE); |
---|
102 | | - else |
---|
103 | | - text_poke_bp((void *)entry->code, &code, JUMP_LABEL_NOP_SIZE, |
---|
104 | | - (void *)entry->code + JUMP_LABEL_NOP_SIZE); |
---|
| 78 | + if (init || system_state == SYSTEM_BOOTING) { |
---|
| 79 | + text_poke_early((void *)jump_entry_code(entry), opcode, |
---|
| 80 | + JUMP_LABEL_NOP_SIZE); |
---|
| 81 | + return; |
---|
| 82 | + } |
---|
| 83 | + |
---|
| 84 | + text_poke_bp((void *)jump_entry_code(entry), opcode, JUMP_LABEL_NOP_SIZE, NULL); |
---|
| 85 | +} |
---|
| 86 | + |
---|
| 87 | +static void __ref jump_label_transform(struct jump_entry *entry, |
---|
| 88 | + enum jump_label_type type, |
---|
| 89 | + int init) |
---|
| 90 | +{ |
---|
| 91 | + mutex_lock(&text_mutex); |
---|
| 92 | + __jump_label_transform(entry, type, init); |
---|
| 93 | + mutex_unlock(&text_mutex); |
---|
105 | 94 | } |
---|
106 | 95 | |
---|
107 | 96 | void arch_jump_label_transform(struct jump_entry *entry, |
---|
108 | 97 | enum jump_label_type type) |
---|
109 | 98 | { |
---|
| 99 | + jump_label_transform(entry, type, 0); |
---|
| 100 | +} |
---|
| 101 | + |
---|
| 102 | +bool arch_jump_label_transform_queue(struct jump_entry *entry, |
---|
| 103 | + enum jump_label_type type) |
---|
| 104 | +{ |
---|
| 105 | + const void *opcode; |
---|
| 106 | + |
---|
| 107 | + if (system_state == SYSTEM_BOOTING) { |
---|
| 108 | + /* |
---|
| 109 | + * Fallback to the non-batching mode. |
---|
| 110 | + */ |
---|
| 111 | + arch_jump_label_transform(entry, type); |
---|
| 112 | + return true; |
---|
| 113 | + } |
---|
| 114 | + |
---|
110 | 115 | mutex_lock(&text_mutex); |
---|
111 | | - __jump_label_transform(entry, type, NULL, 0); |
---|
| 116 | + opcode = __jump_label_set_jump_code(entry, type, 0); |
---|
| 117 | + text_poke_queue((void *)jump_entry_code(entry), |
---|
| 118 | + opcode, JUMP_LABEL_NOP_SIZE, NULL); |
---|
| 119 | + mutex_unlock(&text_mutex); |
---|
| 120 | + return true; |
---|
| 121 | +} |
---|
| 122 | + |
---|
| 123 | +void arch_jump_label_transform_apply(void) |
---|
| 124 | +{ |
---|
| 125 | + mutex_lock(&text_mutex); |
---|
| 126 | + text_poke_finish(); |
---|
112 | 127 | mutex_unlock(&text_mutex); |
---|
113 | 128 | } |
---|
114 | 129 | |
---|
.. | .. |
---|
138 | 153 | jlstate = JL_STATE_NO_UPDATE; |
---|
139 | 154 | } |
---|
140 | 155 | if (jlstate == JL_STATE_UPDATE) |
---|
141 | | - __jump_label_transform(entry, type, text_poke_early, 1); |
---|
| 156 | + jump_label_transform(entry, type, 1); |
---|
142 | 157 | } |
---|