forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2023-12-09 b22da3d8526a935aa31e086e63f60ff3246cb61c 
 kernel/include/linux/rwsem.h
....@@ -16,58 +16,63 @@
1616 #include <linux/spinlock.h>
1717 #include <linux/atomic.h>
1818 #include <linux/err.h>
19
+
20
+#ifdef CONFIG_PREEMPT_RT
21
+#include <linux/rwsem-rt.h>
22
+#else /* PREEMPT_RT */
23
+
1924 #ifdef CONFIG_RWSEM_SPIN_ON_OWNER
2025 #include <linux/osq_lock.h>
2126 #endif
27
+#include <linux/android_vendor.h>
2228 
23
-#ifdef CONFIG_PREEMPT_RT_FULL
24
-#include <linux/rwsem_rt.h>
25
-#else /* PREEMPT_RT_FULL */
26
-
27
-struct rw_semaphore;
28
-
29
-#ifdef CONFIG_RWSEM_GENERIC_SPINLOCK
30
-#include <linux/rwsem-spinlock.h> /* use a generic implementation */
31
-#define __RWSEM_INIT_COUNT(name)	.count = RWSEM_UNLOCKED_VALUE
32
-#else
33
-/* All arch specific implementations share the same struct */
29
+/*
30
+ * For an uncontended rwsem, count and owner are the only fields a task
31
+ * needs to touch when acquiring the rwsem. So they are put next to each
32
+ * other to increase the chance that they will share the same cacheline.
33
+ *
34
+ * In a contended rwsem, the owner is likely the most frequently accessed
35
+ * field in the structure as the optimistic waiter that holds the osq lock
36
+ * will spin on owner. For an embedded rwsem, other hot fields in the
37
+ * containing structure should be moved further away from the rwsem to
38
+ * reduce the chance that they will share the same cacheline causing
39
+ * cacheline bouncing problem.
40
+ */
3441 struct rw_semaphore {
3542 	atomic_long_t count;
36
-	struct list_head wait_list;
37
-	raw_spinlock_t wait_lock;
43
+	/*
44
+	 * Write owner or one of the read owners as well flags regarding
45
+	 * the current state of the rwsem. Can be used as a speculative
46
+	 * check to see if the write owner is running on the cpu.
47
+	 */
48
+	atomic_long_t owner;
3849 #ifdef CONFIG_RWSEM_SPIN_ON_OWNER
3950 	struct optimistic_spin_queue osq; /* spinner MCS lock */
40
-	/*
41
-	 * Write owner. Used as a speculative check to see
42
-	 * if the owner is running on the cpu.
43
-	 */
44
-	struct task_struct *owner;
51
+#endif
52
+	raw_spinlock_t wait_lock;
53
+	struct list_head wait_list;
54
+#ifdef CONFIG_DEBUG_RWSEMS
55
+	void *magic;
4556 #endif
4657 #ifdef CONFIG_DEBUG_LOCK_ALLOC
4758 	struct lockdep_map	dep_map;
4859 #endif
49
-        /* NOTICE: m_count is a vendor variable used for the config
50
-         * CONFIG_RWSEM_PRIO_AWARE. This is included here to maintain ABI
51
-         * compatibility with our vendors */
52
-        /* count for waiters preempt to queue in wait list */
53
-	long m_count;
60
+	ANDROID_VENDOR_DATA(1);
61
+	ANDROID_OEM_DATA_ARRAY(1, 2);
5462 };
5563 
56
-/*
57
- * Setting bit 0 of the owner field with other non-zero bits will indicate
58
- * that the rwsem is writer-owned with an unknown owner.
59
- */
60
-#define RWSEM_OWNER_UNKNOWN	((struct task_struct *)-1L)
64
+enum rwsem_waiter_type {
65
+	RWSEM_WAITING_FOR_WRITE,
66
+	RWSEM_WAITING_FOR_READ
67
+};
6168 
62
-extern struct rw_semaphore *rwsem_down_read_failed(struct rw_semaphore *sem);
63
-extern struct rw_semaphore *rwsem_down_read_failed_killable(struct rw_semaphore *sem);
64
-extern struct rw_semaphore *rwsem_down_write_failed(struct rw_semaphore *sem);
65
-extern struct rw_semaphore *rwsem_down_write_failed_killable(struct rw_semaphore *sem);
66
-extern struct rw_semaphore *rwsem_wake(struct rw_semaphore *);
67
-extern struct rw_semaphore *rwsem_downgrade_wake(struct rw_semaphore *sem);
68
-
69
-/* Include the arch specific part */
70
-#include <asm/rwsem.h>
69
+struct rwsem_waiter {
70
+	struct list_head list;
71
+	struct task_struct *task;
72
+	enum rwsem_waiter_type type;
73
+	unsigned long timeout;
74
+	unsigned long last_rowner;
75
+};
7176 
7277 /* In all implementations count != 0 means locked */
7378 static inline int rwsem_is_locked(struct rw_semaphore *sem)
....@@ -75,28 +80,40 @@
7580 	return atomic_long_read(&sem->count) != 0;
7681 }
7782 
78
-#define __RWSEM_INIT_COUNT(name)	.count = ATOMIC_LONG_INIT(RWSEM_UNLOCKED_VALUE)
79
-#endif
83
+#define RWSEM_UNLOCKED_VALUE		0L
84
+#define __RWSEM_COUNT_INIT(name)	.count = ATOMIC_LONG_INIT(RWSEM_UNLOCKED_VALUE)
8085 
8186 /* Common initializer macros and functions */
8287 
8388 #ifdef CONFIG_DEBUG_LOCK_ALLOC
84
-# define __RWSEM_DEP_MAP_INIT(lockname) , .dep_map = { .name = #lockname }
89
+# define __RWSEM_DEP_MAP_INIT(lockname)			\
90
+	.dep_map = {					\
91
+		.name = #lockname,			\
92
+		.wait_type_inner = LD_WAIT_SLEEP,	\
93
+	},
8594 #else
8695 # define __RWSEM_DEP_MAP_INIT(lockname)
8796 #endif
8897 
98
+#ifdef CONFIG_DEBUG_RWSEMS
99
+# define __RWSEM_DEBUG_INIT(lockname) .magic = &lockname,
100
+#else
101
+# define __RWSEM_DEBUG_INIT(lockname)
102
+#endif
103
+
89104 #ifdef CONFIG_RWSEM_SPIN_ON_OWNER
90
-#define __RWSEM_OPT_INIT(lockname) , .osq = OSQ_LOCK_UNLOCKED, .owner = NULL
105
+#define __RWSEM_OPT_INIT(lockname) .osq = OSQ_LOCK_UNLOCKED,
91106 #else
92107 #define __RWSEM_OPT_INIT(lockname)
93108 #endif
94109 
95110 #define __RWSEM_INITIALIZER(name)				\
96
-	{ __RWSEM_INIT_COUNT(name),				\
97
-	  .wait_list = LIST_HEAD_INIT((name).wait_list),	\
98
-	  .wait_lock = __RAW_SPIN_LOCK_UNLOCKED(name.wait_lock)	\
111
+	{ __RWSEM_COUNT_INIT(name),				\
112
+	  .owner = ATOMIC_LONG_INIT(0),				\
99113 	  __RWSEM_OPT_INIT(name)				\
114
+	  .wait_lock = __RAW_SPIN_LOCK_UNLOCKED(name.wait_lock),\
115
+	  .wait_list = LIST_HEAD_INIT((name).wait_list),	\
116
+	  __RWSEM_DEBUG_INIT(name)				\
100117 	  __RWSEM_DEP_MAP_INIT(name) }
101118 
102119 #define DECLARE_RWSEM(name) \
....@@ -123,7 +140,7 @@
123140 	return !list_empty(&sem->wait_list);
124141 }
125142 
126
-#endif /* !PREEMPT_RT_FULL */
143
+#endif /* !PREEMPT_RT */
127144 
128145 /*
129146  * The functions below are the same for all rwsem implementations including
....@@ -134,6 +151,7 @@
134151  * lock for reading
135152  */
136153 extern void down_read(struct rw_semaphore *sem);
154
+extern int __must_check down_read_interruptible(struct rw_semaphore *sem);
137155 extern int __must_check down_read_killable(struct rw_semaphore *sem);
138156 
139157 /*
....@@ -179,9 +197,10 @@
179197  * static then another method for expressing nested locking is
180198  * the explicit definition of lock class keys and the use of
181199  * lockdep_set_class() at lock initialization time.
182
- * See Documentation/locking/lockdep-design.txt for more details.)
200
+ * See Documentation/locking/lockdep-design.rst for more details.)
183201  */
184202 extern void down_read_nested(struct rw_semaphore *sem, int subclass);
203
+extern int __must_check down_read_killable_nested(struct rw_semaphore *sem, int subclass);
185204 extern void down_write_nested(struct rw_semaphore *sem, int subclass);
186205 extern int down_write_killable_nested(struct rw_semaphore *sem, int subclass);
187206 extern void _down_write_nest_lock(struct rw_semaphore *sem, struct lockdep_map *nest_lock);
....@@ -202,6 +221,7 @@
202221 extern void up_read_non_owner(struct rw_semaphore *sem);
203222 #else
204223 # define down_read_nested(sem, subclass)		down_read(sem)
224
+# define down_read_killable_nested(sem, subclass)	down_read_killable(sem)
205225 # define down_write_nest_lock(sem, nest_lock)	down_write(sem)
206226 # define down_write_nested(sem, subclass)	down_write(sem)
207227 # define down_write_killable_nested(sem, subclass)	down_write_killable(sem)