hc
2023-12-09 b22da3d8526a935aa31e086e63f60ff3246cb61c
kernel/include/linux/jump_label.h
....@@ -68,7 +68,7 @@
6868 * Lacking toolchain and or architecture support, static keys fall back to a
6969 * simple conditional branch.
7070 *
71
- * Additional babbling in: Documentation/static-keys.txt
71
+ * Additional babbling in: Documentation/staging/static-keys.rst
7272 */
7373
7474 #ifndef __ASSEMBLY__
....@@ -113,8 +113,70 @@
113113 #endif /* CONFIG_JUMP_LABEL */
114114 #endif /* __ASSEMBLY__ */
115115
116
-#ifdef CONFIG_JUMP_LABEL
116
+#if defined(CONFIG_JUMP_LABEL) && !defined(BUILD_FIPS140_KO)
117117 #include <asm/jump_label.h>
118
+
119
+#ifndef __ASSEMBLY__
120
+#ifdef CONFIG_HAVE_ARCH_JUMP_LABEL_RELATIVE
121
+
122
+struct jump_entry {
123
+ s32 code;
124
+ s32 target;
125
+ long key; // key may be far away from the core kernel under KASLR
126
+};
127
+
128
+static inline unsigned long jump_entry_code(const struct jump_entry *entry)
129
+{
130
+ return (unsigned long)&entry->code + entry->code;
131
+}
132
+
133
+static inline unsigned long jump_entry_target(const struct jump_entry *entry)
134
+{
135
+ return (unsigned long)&entry->target + entry->target;
136
+}
137
+
138
+static inline struct static_key *jump_entry_key(const struct jump_entry *entry)
139
+{
140
+ long offset = entry->key & ~3L;
141
+
142
+ return (struct static_key *)((unsigned long)&entry->key + offset);
143
+}
144
+
145
+#else
146
+
147
+static inline unsigned long jump_entry_code(const struct jump_entry *entry)
148
+{
149
+ return entry->code;
150
+}
151
+
152
+static inline unsigned long jump_entry_target(const struct jump_entry *entry)
153
+{
154
+ return entry->target;
155
+}
156
+
157
+static inline struct static_key *jump_entry_key(const struct jump_entry *entry)
158
+{
159
+ return (struct static_key *)((unsigned long)entry->key & ~3UL);
160
+}
161
+
162
+#endif
163
+
164
+static inline bool jump_entry_is_branch(const struct jump_entry *entry)
165
+{
166
+ return (unsigned long)entry->key & 1UL;
167
+}
168
+
169
+static inline bool jump_entry_is_init(const struct jump_entry *entry)
170
+{
171
+ return (unsigned long)entry->key & 2UL;
172
+}
173
+
174
+static inline void jump_entry_set_init(struct jump_entry *entry)
175
+{
176
+ entry->key |= 2;
177
+}
178
+
179
+#endif
118180 #endif
119181
120182 #ifndef __ASSEMBLY__
....@@ -126,7 +188,30 @@
126188
127189 struct module;
128190
129
-#ifdef CONFIG_JUMP_LABEL
191
+#ifdef BUILD_FIPS140_KO
192
+
193
+#include <linux/atomic.h>
194
+
195
+static inline int static_key_count(struct static_key *key)
196
+{
197
+ return atomic_read(&key->enabled);
198
+}
199
+
200
+static __always_inline bool static_key_false(struct static_key *key)
201
+{
202
+ if (unlikely(static_key_count(key) > 0))
203
+ return true;
204
+ return false;
205
+}
206
+
207
+static __always_inline bool static_key_true(struct static_key *key)
208
+{
209
+ if (likely(static_key_count(key) > 0))
210
+ return true;
211
+ return false;
212
+}
213
+
214
+#elif defined(CONFIG_JUMP_LABEL)
130215
131216 #define JUMP_TYPE_FALSE 0UL
132217 #define JUMP_TYPE_TRUE 1UL
....@@ -147,13 +232,15 @@
147232 extern struct jump_entry __stop___jump_table[];
148233
149234 extern void jump_label_init(void);
150
-extern void jump_label_invalidate_initmem(void);
151235 extern void jump_label_lock(void);
152236 extern void jump_label_unlock(void);
153237 extern void arch_jump_label_transform(struct jump_entry *entry,
154238 enum jump_label_type type);
155239 extern void arch_jump_label_transform_static(struct jump_entry *entry,
156240 enum jump_label_type type);
241
+extern bool arch_jump_label_transform_queue(struct jump_entry *entry,
242
+ enum jump_label_type type);
243
+extern void arch_jump_label_transform_apply(void);
157244 extern int jump_label_text_reserved(void *start, void *end);
158245 extern void static_key_slow_inc(struct static_key *key);
159246 extern void static_key_slow_dec(struct static_key *key);
....@@ -185,17 +272,15 @@
185272 #include <linux/atomic.h>
186273 #include <linux/bug.h>
187274
188
-static inline int static_key_count(struct static_key *key)
275
+static __always_inline int static_key_count(struct static_key *key)
189276 {
190
- return atomic_read(&key->enabled);
277
+ return arch_atomic_read(&key->enabled);
191278 }
192279
193280 static __always_inline void jump_label_init(void)
194281 {
195282 static_key_initialized = true;
196283 }
197
-
198
-static inline void jump_label_invalidate_initmem(void) {}
199284
200285 static __always_inline bool static_key_false(struct static_key *key)
201286 {
....@@ -331,7 +416,7 @@
331416 static_key_count((struct static_key *)x) > 0; \
332417 })
333418
334
-#ifdef CONFIG_JUMP_LABEL
419
+#if defined(CONFIG_JUMP_LABEL) && !defined(BUILD_FIPS140_KO)
335420
336421 /*
337422 * Combine the right initial value (type) with the right branch order