| .. | .. |
|---|
| 68 | 68 | * Lacking toolchain and or architecture support, static keys fall back to a |
|---|
| 69 | 69 | * simple conditional branch. |
|---|
| 70 | 70 | * |
|---|
| 71 | | - * Additional babbling in: Documentation/static-keys.txt |
|---|
| 71 | + * Additional babbling in: Documentation/staging/static-keys.rst |
|---|
| 72 | 72 | */ |
|---|
| 73 | 73 | |
|---|
| 74 | 74 | #ifndef __ASSEMBLY__ |
|---|
| .. | .. |
|---|
| 113 | 113 | #endif /* CONFIG_JUMP_LABEL */ |
|---|
| 114 | 114 | #endif /* __ASSEMBLY__ */ |
|---|
| 115 | 115 | |
|---|
| 116 | | -#ifdef CONFIG_JUMP_LABEL |
|---|
| 116 | +#if defined(CONFIG_JUMP_LABEL) && !defined(BUILD_FIPS140_KO) |
|---|
| 117 | 117 | #include <asm/jump_label.h> |
|---|
| 118 | + |
|---|
| 119 | +#ifndef __ASSEMBLY__ |
|---|
| 120 | +#ifdef CONFIG_HAVE_ARCH_JUMP_LABEL_RELATIVE |
|---|
| 121 | + |
|---|
| 122 | +struct jump_entry { |
|---|
| 123 | + s32 code; |
|---|
| 124 | + s32 target; |
|---|
| 125 | + long key; // key may be far away from the core kernel under KASLR |
|---|
| 126 | +}; |
|---|
| 127 | + |
|---|
| 128 | +static inline unsigned long jump_entry_code(const struct jump_entry *entry) |
|---|
| 129 | +{ |
|---|
| 130 | + return (unsigned long)&entry->code + entry->code; |
|---|
| 131 | +} |
|---|
| 132 | + |
|---|
| 133 | +static inline unsigned long jump_entry_target(const struct jump_entry *entry) |
|---|
| 134 | +{ |
|---|
| 135 | + return (unsigned long)&entry->target + entry->target; |
|---|
| 136 | +} |
|---|
| 137 | + |
|---|
| 138 | +static inline struct static_key *jump_entry_key(const struct jump_entry *entry) |
|---|
| 139 | +{ |
|---|
| 140 | + long offset = entry->key & ~3L; |
|---|
| 141 | + |
|---|
| 142 | + return (struct static_key *)((unsigned long)&entry->key + offset); |
|---|
| 143 | +} |
|---|
| 144 | + |
|---|
| 145 | +#else |
|---|
| 146 | + |
|---|
| 147 | +static inline unsigned long jump_entry_code(const struct jump_entry *entry) |
|---|
| 148 | +{ |
|---|
| 149 | + return entry->code; |
|---|
| 150 | +} |
|---|
| 151 | + |
|---|
| 152 | +static inline unsigned long jump_entry_target(const struct jump_entry *entry) |
|---|
| 153 | +{ |
|---|
| 154 | + return entry->target; |
|---|
| 155 | +} |
|---|
| 156 | + |
|---|
| 157 | +static inline struct static_key *jump_entry_key(const struct jump_entry *entry) |
|---|
| 158 | +{ |
|---|
| 159 | + return (struct static_key *)((unsigned long)entry->key & ~3UL); |
|---|
| 160 | +} |
|---|
| 161 | + |
|---|
| 162 | +#endif |
|---|
| 163 | + |
|---|
| 164 | +static inline bool jump_entry_is_branch(const struct jump_entry *entry) |
|---|
| 165 | +{ |
|---|
| 166 | + return (unsigned long)entry->key & 1UL; |
|---|
| 167 | +} |
|---|
| 168 | + |
|---|
| 169 | +static inline bool jump_entry_is_init(const struct jump_entry *entry) |
|---|
| 170 | +{ |
|---|
| 171 | + return (unsigned long)entry->key & 2UL; |
|---|
| 172 | +} |
|---|
| 173 | + |
|---|
| 174 | +static inline void jump_entry_set_init(struct jump_entry *entry) |
|---|
| 175 | +{ |
|---|
| 176 | + entry->key |= 2; |
|---|
| 177 | +} |
|---|
| 178 | + |
|---|
| 179 | +#endif |
|---|
| 118 | 180 | #endif |
|---|
| 119 | 181 | |
|---|
| 120 | 182 | #ifndef __ASSEMBLY__ |
|---|
| .. | .. |
|---|
| 126 | 188 | |
|---|
| 127 | 189 | struct module; |
|---|
| 128 | 190 | |
|---|
| 129 | | -#ifdef CONFIG_JUMP_LABEL |
|---|
| 191 | +#ifdef BUILD_FIPS140_KO |
|---|
| 192 | + |
|---|
| 193 | +#include <linux/atomic.h> |
|---|
| 194 | + |
|---|
| 195 | +static inline int static_key_count(struct static_key *key) |
|---|
| 196 | +{ |
|---|
| 197 | + return atomic_read(&key->enabled); |
|---|
| 198 | +} |
|---|
| 199 | + |
|---|
| 200 | +static __always_inline bool static_key_false(struct static_key *key) |
|---|
| 201 | +{ |
|---|
| 202 | + if (unlikely(static_key_count(key) > 0)) |
|---|
| 203 | + return true; |
|---|
| 204 | + return false; |
|---|
| 205 | +} |
|---|
| 206 | + |
|---|
| 207 | +static __always_inline bool static_key_true(struct static_key *key) |
|---|
| 208 | +{ |
|---|
| 209 | + if (likely(static_key_count(key) > 0)) |
|---|
| 210 | + return true; |
|---|
| 211 | + return false; |
|---|
| 212 | +} |
|---|
| 213 | + |
|---|
| 214 | +#elif defined(CONFIG_JUMP_LABEL) |
|---|
| 130 | 215 | |
|---|
| 131 | 216 | #define JUMP_TYPE_FALSE 0UL |
|---|
| 132 | 217 | #define JUMP_TYPE_TRUE 1UL |
|---|
| .. | .. |
|---|
| 147 | 232 | extern struct jump_entry __stop___jump_table[]; |
|---|
| 148 | 233 | |
|---|
| 149 | 234 | extern void jump_label_init(void); |
|---|
| 150 | | -extern void jump_label_invalidate_initmem(void); |
|---|
| 151 | 235 | extern void jump_label_lock(void); |
|---|
| 152 | 236 | extern void jump_label_unlock(void); |
|---|
| 153 | 237 | extern void arch_jump_label_transform(struct jump_entry *entry, |
|---|
| 154 | 238 | enum jump_label_type type); |
|---|
| 155 | 239 | extern void arch_jump_label_transform_static(struct jump_entry *entry, |
|---|
| 156 | 240 | enum jump_label_type type); |
|---|
| 241 | +extern bool arch_jump_label_transform_queue(struct jump_entry *entry, |
|---|
| 242 | + enum jump_label_type type); |
|---|
| 243 | +extern void arch_jump_label_transform_apply(void); |
|---|
| 157 | 244 | extern int jump_label_text_reserved(void *start, void *end); |
|---|
| 158 | 245 | extern void static_key_slow_inc(struct static_key *key); |
|---|
| 159 | 246 | extern void static_key_slow_dec(struct static_key *key); |
|---|
| .. | .. |
|---|
| 185 | 272 | #include <linux/atomic.h> |
|---|
| 186 | 273 | #include <linux/bug.h> |
|---|
| 187 | 274 | |
|---|
| 188 | | -static inline int static_key_count(struct static_key *key) |
|---|
| 275 | +static __always_inline int static_key_count(struct static_key *key) |
|---|
| 189 | 276 | { |
|---|
| 190 | | - return atomic_read(&key->enabled); |
|---|
| 277 | + return arch_atomic_read(&key->enabled); |
|---|
| 191 | 278 | } |
|---|
| 192 | 279 | |
|---|
| 193 | 280 | static __always_inline void jump_label_init(void) |
|---|
| 194 | 281 | { |
|---|
| 195 | 282 | static_key_initialized = true; |
|---|
| 196 | 283 | } |
|---|
| 197 | | - |
|---|
| 198 | | -static inline void jump_label_invalidate_initmem(void) {} |
|---|
| 199 | 284 | |
|---|
| 200 | 285 | static __always_inline bool static_key_false(struct static_key *key) |
|---|
| 201 | 286 | { |
|---|
| .. | .. |
|---|
| 331 | 416 | static_key_count((struct static_key *)x) > 0; \ |
|---|
| 332 | 417 | }) |
|---|
| 333 | 418 | |
|---|
| 334 | | -#ifdef CONFIG_JUMP_LABEL |
|---|
| 419 | +#if defined(CONFIG_JUMP_LABEL) && !defined(BUILD_FIPS140_KO) |
|---|
| 335 | 420 | |
|---|
| 336 | 421 | /* |
|---|
| 337 | 422 | * Combine the right initial value (type) with the right branch order |
|---|