.. | .. |
---|
56 | 56 | #include <linux/kernel.h> |
---|
57 | 57 | #include <linux/stringify.h> |
---|
58 | 58 | #include <linux/bottom_half.h> |
---|
| 59 | +#include <linux/lockdep.h> |
---|
59 | 60 | #include <asm/barrier.h> |
---|
| 61 | +#include <asm/mmiowb.h> |
---|
60 | 62 | |
---|
61 | 63 | |
---|
62 | 64 | /* |
---|
.. | .. |
---|
74 | 76 | #define LOCK_SECTION_END \ |
---|
75 | 77 | ".previous\n\t" |
---|
76 | 78 | |
---|
77 | | -#define __lockfunc __attribute__((section(".spinlock.text"))) |
---|
| 79 | +#define __lockfunc __section(".spinlock.text") |
---|
78 | 80 | |
---|
79 | 81 | /* |
---|
80 | 82 | * Pull the arch_spinlock_t and arch_rwlock_t definitions: |
---|
.. | .. |
---|
92 | 94 | |
---|
93 | 95 | #ifdef CONFIG_DEBUG_SPINLOCK |
---|
94 | 96 | extern void __raw_spin_lock_init(raw_spinlock_t *lock, const char *name, |
---|
95 | | - struct lock_class_key *key); |
---|
96 | | -# define raw_spin_lock_init(lock) \ |
---|
97 | | -do { \ |
---|
98 | | - static struct lock_class_key __key; \ |
---|
99 | | - \ |
---|
100 | | - __raw_spin_lock_init((lock), #lock, &__key); \ |
---|
| 97 | + struct lock_class_key *key, short inner); |
---|
| 98 | + |
---|
| 99 | +# define raw_spin_lock_init(lock) \ |
---|
| 100 | +do { \ |
---|
| 101 | + static struct lock_class_key __key; \ |
---|
| 102 | + \ |
---|
| 103 | + __raw_spin_lock_init((lock), #lock, &__key, LD_WAIT_SPIN); \ |
---|
101 | 104 | } while (0) |
---|
102 | 105 | |
---|
103 | 106 | #else |
---|
.. | .. |
---|
178 | 181 | { |
---|
179 | 182 | __acquire(lock); |
---|
180 | 183 | arch_spin_lock(&lock->raw_lock); |
---|
| 184 | + mmiowb_spin_lock(); |
---|
181 | 185 | } |
---|
182 | 186 | |
---|
183 | 187 | #ifndef arch_spin_lock_flags |
---|
.. | .. |
---|
189 | 193 | { |
---|
190 | 194 | __acquire(lock); |
---|
191 | 195 | arch_spin_lock_flags(&lock->raw_lock, *flags); |
---|
| 196 | + mmiowb_spin_lock(); |
---|
192 | 197 | } |
---|
193 | 198 | |
---|
194 | 199 | static inline int do_raw_spin_trylock(raw_spinlock_t *lock) |
---|
195 | 200 | { |
---|
196 | | - return arch_spin_trylock(&(lock)->raw_lock); |
---|
| 201 | + int ret = arch_spin_trylock(&(lock)->raw_lock); |
---|
| 202 | + |
---|
| 203 | + if (ret) |
---|
| 204 | + mmiowb_spin_lock(); |
---|
| 205 | + |
---|
| 206 | + return ret; |
---|
197 | 207 | } |
---|
198 | 208 | |
---|
199 | 209 | static inline void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock) |
---|
200 | 210 | { |
---|
| 211 | + mmiowb_spin_unlock(); |
---|
201 | 212 | arch_spin_unlock(&lock->raw_lock); |
---|
202 | 213 | __release(lock); |
---|
203 | 214 | } |
---|
.. | .. |
---|
205 | 216 | |
---|
206 | 217 | /* |
---|
207 | 218 | * Define the various spin_lock methods. Note we define these |
---|
208 | | - * regardless of whether CONFIG_SMP or CONFIG_PREEMPT are set. The |
---|
| 219 | + * regardless of whether CONFIG_SMP or CONFIG_PREEMPTION are set. The |
---|
209 | 220 | * various methods are defined as nops in the case they are not |
---|
210 | 221 | * required. |
---|
211 | 222 | */ |
---|
.. | .. |
---|
298 | 309 | }) |
---|
299 | 310 | |
---|
300 | 311 | /* Include rwlock functions */ |
---|
301 | | -#ifdef CONFIG_PREEMPT_RT_FULL |
---|
302 | | -# include <linux/rwlock_rt.h> |
---|
303 | | -#else |
---|
304 | | -# include <linux/rwlock.h> |
---|
305 | | -#endif |
---|
| 312 | +#include <linux/rwlock.h> |
---|
306 | 313 | |
---|
307 | 314 | /* |
---|
308 | 315 | * Pull the _spin_*()/_read_*()/_write_*() functions/declarations: |
---|
.. | .. |
---|
313 | 320 | # include <linux/spinlock_api_up.h> |
---|
314 | 321 | #endif |
---|
315 | 322 | |
---|
316 | | -#ifdef CONFIG_PREEMPT_RT_FULL |
---|
317 | | -# include <linux/spinlock_rt.h> |
---|
318 | | -#else /* PREEMPT_RT_FULL */ |
---|
319 | | - |
---|
320 | 323 | /* |
---|
321 | 324 | * Map the spin_lock functions to the raw variants for PREEMPT_RT=n |
---|
322 | 325 | */ |
---|
.. | .. |
---|
326 | 329 | return &lock->rlock; |
---|
327 | 330 | } |
---|
328 | 331 | |
---|
329 | | -#define spin_lock_init(_lock) \ |
---|
330 | | -do { \ |
---|
331 | | - spinlock_check(_lock); \ |
---|
332 | | - raw_spin_lock_init(&(_lock)->rlock); \ |
---|
| 332 | +#ifdef CONFIG_DEBUG_SPINLOCK |
---|
| 333 | + |
---|
| 334 | +# define spin_lock_init(lock) \ |
---|
| 335 | +do { \ |
---|
| 336 | + static struct lock_class_key __key; \ |
---|
| 337 | + \ |
---|
| 338 | + __raw_spin_lock_init(spinlock_check(lock), \ |
---|
| 339 | + #lock, &__key, LD_WAIT_CONFIG); \ |
---|
333 | 340 | } while (0) |
---|
| 341 | + |
---|
| 342 | +#else |
---|
| 343 | + |
---|
| 344 | +# define spin_lock_init(_lock) \ |
---|
| 345 | +do { \ |
---|
| 346 | + spinlock_check(_lock); \ |
---|
| 347 | + *(_lock) = __SPIN_LOCK_UNLOCKED(_lock); \ |
---|
| 348 | +} while (0) |
---|
| 349 | + |
---|
| 350 | +#endif |
---|
334 | 351 | |
---|
335 | 352 | static __always_inline void spin_lock(spinlock_t *lock) |
---|
336 | 353 | { |
---|
.. | .. |
---|
436 | 453 | } |
---|
437 | 454 | |
---|
438 | 455 | #define assert_spin_locked(lock) assert_raw_spin_locked(&(lock)->rlock) |
---|
439 | | - |
---|
440 | | -#endif /* !PREEMPT_RT_FULL */ |
---|
441 | 456 | |
---|
442 | 457 | /* |
---|
443 | 458 | * Pull the atomic_t declaration: |
---|