| .. | .. |
|---|
| 1 | | -/* SPDX-License-Identifier: GPL-2.0 */ |
|---|
| 2 | | -/* |
|---|
| 3 | | - * The owner field of the rw_semaphore structure will be set to |
|---|
| 4 | | - * RWSEM_READER_OWNED when a reader grabs the lock. A writer will clear |
|---|
| 5 | | - * the owner field when it unlocks. A reader, on the other hand, will |
|---|
| 6 | | - * not touch the owner field when it unlocks. |
|---|
| 7 | | - * |
|---|
| 8 | | - * In essence, the owner field now has the following 4 states: |
|---|
| 9 | | - * 1) 0 |
|---|
| 10 | | - * - lock is free or the owner hasn't set the field yet |
|---|
| 11 | | - * 2) RWSEM_READER_OWNED |
|---|
| 12 | | - * - lock is currently or previously owned by readers (lock is free |
|---|
| 13 | | - * or not set by owner yet) |
|---|
| 14 | | - * 3) RWSEM_ANONYMOUSLY_OWNED bit set with some other bits set as well |
|---|
| 15 | | - * - lock is owned by an anonymous writer, so spinning on the lock |
|---|
| 16 | | - * owner should be disabled. |
|---|
| 17 | | - * 4) Other non-zero value |
|---|
| 18 | | - * - a writer owns the lock and other writers can spin on the lock owner. |
|---|
| 19 | | - */ |
|---|
| 20 | | -#define RWSEM_ANONYMOUSLY_OWNED (1UL << 0) |
|---|
| 21 | | -#define RWSEM_READER_OWNED ((struct task_struct *)RWSEM_ANONYMOUSLY_OWNED) |
|---|
| 22 | | - |
|---|
| 23 | | -#ifdef CONFIG_DEBUG_RWSEMS |
|---|
| 24 | | -# define DEBUG_RWSEMS_WARN_ON(c) DEBUG_LOCKS_WARN_ON(c) |
|---|
| 25 | | -#else |
|---|
| 26 | | -# define DEBUG_RWSEMS_WARN_ON(c) |
|---|
| 27 | | -#endif |
|---|
| 28 | | - |
|---|
| 29 | | -#ifdef CONFIG_RWSEM_SPIN_ON_OWNER |
|---|
| 30 | | -/* |
|---|
| 31 | | - * All writes to owner are protected by WRITE_ONCE() to make sure that |
|---|
| 32 | | - * store tearing can't happen as optimistic spinners may read and use |
|---|
| 33 | | - * the owner value concurrently without lock. Read from owner, however, |
|---|
| 34 | | - * may not need READ_ONCE() as long as the pointer value is only used |
|---|
| 35 | | - * for comparison and isn't being dereferenced. |
|---|
| 36 | | - */ |
|---|
| 37 | | -static inline void rwsem_set_owner(struct rw_semaphore *sem) |
|---|
| 38 | | -{ |
|---|
| 39 | | - WRITE_ONCE(sem->owner, current); |
|---|
| 40 | | -} |
|---|
| 41 | | - |
|---|
| 42 | | -static inline void rwsem_clear_owner(struct rw_semaphore *sem) |
|---|
| 43 | | -{ |
|---|
| 44 | | - WRITE_ONCE(sem->owner, NULL); |
|---|
| 45 | | -} |
|---|
| 46 | | - |
|---|
| 47 | | -static inline void rwsem_set_reader_owned(struct rw_semaphore *sem) |
|---|
| 48 | | -{ |
|---|
| 49 | | - /* |
|---|
| 50 | | - * We check the owner value first to make sure that we will only |
|---|
| 51 | | - * do a write to the rwsem cacheline when it is really necessary |
|---|
| 52 | | - * to minimize cacheline contention. |
|---|
| 53 | | - */ |
|---|
| 54 | | - if (READ_ONCE(sem->owner) != RWSEM_READER_OWNED) |
|---|
| 55 | | - WRITE_ONCE(sem->owner, RWSEM_READER_OWNED); |
|---|
| 56 | | -} |
|---|
| 57 | | - |
|---|
| 58 | | -/* |
|---|
| 59 | | - * Return true if the a rwsem waiter can spin on the rwsem's owner |
|---|
| 60 | | - * and steal the lock, i.e. the lock is not anonymously owned. |
|---|
| 61 | | - * N.B. !owner is considered spinnable. |
|---|
| 62 | | - */ |
|---|
| 63 | | -static inline bool is_rwsem_owner_spinnable(struct task_struct *owner) |
|---|
| 64 | | -{ |
|---|
| 65 | | - return !((unsigned long)owner & RWSEM_ANONYMOUSLY_OWNED); |
|---|
| 66 | | -} |
|---|
| 67 | | - |
|---|
| 68 | | -/* |
|---|
| 69 | | - * Return true if rwsem is owned by an anonymous writer or readers. |
|---|
| 70 | | - */ |
|---|
| 71 | | -static inline bool rwsem_has_anonymous_owner(struct task_struct *owner) |
|---|
| 72 | | -{ |
|---|
| 73 | | - return (unsigned long)owner & RWSEM_ANONYMOUSLY_OWNED; |
|---|
| 74 | | -} |
|---|
| 75 | | -#else |
|---|
| 76 | | -static inline void rwsem_set_owner(struct rw_semaphore *sem) |
|---|
| 77 | | -{ |
|---|
| 78 | | -} |
|---|
| 79 | | - |
|---|
| 80 | | -static inline void rwsem_clear_owner(struct rw_semaphore *sem) |
|---|
| 81 | | -{ |
|---|
| 82 | | -} |
|---|
| 83 | | - |
|---|
| 84 | | -static inline void rwsem_set_reader_owned(struct rw_semaphore *sem) |
|---|
| 85 | | -{ |
|---|
| 86 | | -} |
|---|
| 87 | | -#endif |
|---|