| .. | .. | 
|---|
| 1 |  | -/* SPDX-License-Identifier: GPL-2.0 */  | 
|---|
| 2 |  | -/*  | 
|---|
| 3 |  | - * The owner field of the rw_semaphore structure will be set to  | 
|---|
| 4 |  | - * RWSEM_READER_OWNED when a reader grabs the lock. A writer will clear  | 
|---|
| 5 |  | - * the owner field when it unlocks. A reader, on the other hand, will  | 
|---|
| 6 |  | - * not touch the owner field when it unlocks.  | 
|---|
| 7 |  | - *  | 
|---|
| 8 |  | - * In essence, the owner field now has the following 4 states:  | 
|---|
| 9 |  | - *  1) 0  | 
|---|
| 10 |  | - *     - lock is free or the owner hasn't set the field yet  | 
|---|
| 11 |  | - *  2) RWSEM_READER_OWNED  | 
|---|
| 12 |  | - *     - lock is currently or previously owned by readers (lock is free  | 
|---|
| 13 |  | - *       or not set by owner yet)  | 
|---|
| 14 |  | - *  3) RWSEM_ANONYMOUSLY_OWNED bit set with some other bits set as well  | 
|---|
| 15 |  | - *     - lock is owned by an anonymous writer, so spinning on the lock  | 
|---|
| 16 |  | - *       owner should be disabled.  | 
|---|
| 17 |  | - *  4) Other non-zero value  | 
|---|
| 18 |  | - *     - a writer owns the lock and other writers can spin on the lock owner.  | 
|---|
| 19 |  | - */  | 
|---|
| 20 |  | -#define RWSEM_ANONYMOUSLY_OWNED	(1UL << 0)  | 
|---|
| 21 |  | -#define RWSEM_READER_OWNED	((struct task_struct *)RWSEM_ANONYMOUSLY_OWNED)  | 
|---|
| 22 |  | -  | 
|---|
| 23 |  | -#ifdef CONFIG_DEBUG_RWSEMS  | 
|---|
| 24 |  | -# define DEBUG_RWSEMS_WARN_ON(c)	DEBUG_LOCKS_WARN_ON(c)  | 
|---|
| 25 |  | -#else  | 
|---|
| 26 |  | -# define DEBUG_RWSEMS_WARN_ON(c)  | 
|---|
| 27 |  | -#endif  | 
|---|
| 28 |  | -  | 
|---|
| 29 |  | -#ifdef CONFIG_RWSEM_SPIN_ON_OWNER  | 
|---|
| 30 |  | -/*  | 
|---|
| 31 |  | - * All writes to owner are protected by WRITE_ONCE() to make sure that  | 
|---|
| 32 |  | - * store tearing can't happen as optimistic spinners may read and use  | 
|---|
| 33 |  | - * the owner value concurrently without lock. Read from owner, however,  | 
|---|
| 34 |  | - * may not need READ_ONCE() as long as the pointer value is only used  | 
|---|
| 35 |  | - * for comparison and isn't being dereferenced.  | 
|---|
| 36 |  | - */  | 
|---|
| 37 |  | -static inline void rwsem_set_owner(struct rw_semaphore *sem)  | 
|---|
| 38 |  | -{  | 
|---|
| 39 |  | -	WRITE_ONCE(sem->owner, current);  | 
|---|
| 40 |  | -}  | 
|---|
| 41 |  | -  | 
|---|
| 42 |  | -static inline void rwsem_clear_owner(struct rw_semaphore *sem)  | 
|---|
| 43 |  | -{  | 
|---|
| 44 |  | -	WRITE_ONCE(sem->owner, NULL);  | 
|---|
| 45 |  | -}  | 
|---|
| 46 |  | -  | 
|---|
| 47 |  | -static inline void rwsem_set_reader_owned(struct rw_semaphore *sem)  | 
|---|
| 48 |  | -{  | 
|---|
| 49 |  | -	/*  | 
|---|
| 50 |  | -	 * We check the owner value first to make sure that we will only  | 
|---|
| 51 |  | -	 * do a write to the rwsem cacheline when it is really necessary  | 
|---|
| 52 |  | -	 * to minimize cacheline contention.  | 
|---|
| 53 |  | -	 */  | 
|---|
| 54 |  | -	if (READ_ONCE(sem->owner) != RWSEM_READER_OWNED)  | 
|---|
| 55 |  | -		WRITE_ONCE(sem->owner, RWSEM_READER_OWNED);  | 
|---|
| 56 |  | -}  | 
|---|
| 57 |  | -  | 
|---|
| 58 |  | -/*  | 
|---|
| 59 |  | - * Return true if the a rwsem waiter can spin on the rwsem's owner  | 
|---|
| 60 |  | - * and steal the lock, i.e. the lock is not anonymously owned.  | 
|---|
| 61 |  | - * N.B. !owner is considered spinnable.  | 
|---|
| 62 |  | - */  | 
|---|
| 63 |  | -static inline bool is_rwsem_owner_spinnable(struct task_struct *owner)  | 
|---|
| 64 |  | -{  | 
|---|
| 65 |  | -	return !((unsigned long)owner & RWSEM_ANONYMOUSLY_OWNED);  | 
|---|
| 66 |  | -}  | 
|---|
| 67 |  | -  | 
|---|
| 68 |  | -/*  | 
|---|
| 69 |  | - * Return true if rwsem is owned by an anonymous writer or readers.  | 
|---|
| 70 |  | - */  | 
|---|
| 71 |  | -static inline bool rwsem_has_anonymous_owner(struct task_struct *owner)  | 
|---|
| 72 |  | -{  | 
|---|
| 73 |  | -	return (unsigned long)owner & RWSEM_ANONYMOUSLY_OWNED;  | 
|---|
| 74 |  | -}  | 
|---|
| 75 |  | -#else  | 
|---|
| 76 |  | -static inline void rwsem_set_owner(struct rw_semaphore *sem)  | 
|---|
| 77 |  | -{  | 
|---|
| 78 |  | -}  | 
|---|
| 79 |  | -  | 
|---|
| 80 |  | -static inline void rwsem_clear_owner(struct rw_semaphore *sem)  | 
|---|
| 81 |  | -{  | 
|---|
| 82 |  | -}  | 
|---|
| 83 |  | -  | 
|---|
| 84 |  | -static inline void rwsem_set_reader_owned(struct rw_semaphore *sem)  | 
|---|
| 85 |  | -{  | 
|---|
| 86 |  | -}  | 
|---|
| 87 |  | -#endif  | 
|---|