forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2023-12-11 d2ccde1c8e90d38cee87a1b0309ad2827f3fd30d 
 kernel/kernel/locking/mutex.c
....@@ -1,3 +1,4 @@
1
+// SPDX-License-Identifier: GPL-2.0-only
12 /*
23  * kernel/locking/mutex.c
34  *
....@@ -15,7 +16,7 @@
1516  *    by Steven Rostedt, based on work by Gregory Haskins, Peter Morreale
1617  *    and Sven Dietrich.
1718  *
18
- * Also see Documentation/locking/mutex-design.txt.
19
+ * Also see Documentation/locking/mutex-design.rst.
1920  */
2021 #include <linux/mutex.h>
2122 #include <linux/ww_mutex.h>
....@@ -34,6 +35,8 @@
3435 #else
3536 # include "mutex.h"
3637 #endif
38
+
39
+#include <trace/hooks/dtask.h>
3740 
3841 void
3942 __mutex_init(struct mutex *lock, const char *name, struct lock_class_key *key)
....@@ -64,10 +67,36 @@
6467 
6568 #define MUTEX_FLAGS		0x07
6669 
70
+/*
71
+ * Internal helper function; C doesn't allow us to hide it :/
72
+ *
73
+ * DO NOT USE (outside of mutex code).
74
+ */
75
+static inline struct task_struct *__mutex_owner(struct mutex *lock)
76
+{
77
+	return (struct task_struct *)(atomic_long_read(&lock->owner) & ~MUTEX_FLAGS);
78
+}
79
+
6780 static inline struct task_struct *__owner_task(unsigned long owner)
6881 {
6982 	return (struct task_struct *)(owner & ~MUTEX_FLAGS);
7083 }
84
+
85
+bool mutex_is_locked(struct mutex *lock)
86
+{
87
+	return __mutex_owner(lock) != NULL;
88
+}
89
+EXPORT_SYMBOL(mutex_is_locked);
90
+
91
+__must_check enum mutex_trylock_recursive_enum
92
+mutex_trylock_recursive(struct mutex *lock)
93
+{
94
+	if (unlikely(__mutex_owner(lock) == current))
95
+		return MUTEX_TRYLOCK_RECURSIVE;
96
+
97
+	return mutex_trylock(lock);
98
+}
99
+EXPORT_SYMBOL(mutex_trylock_recursive);
71100 
72101 static inline unsigned long __owner_flags(unsigned long owner)
73102 {
....@@ -141,8 +170,10 @@
141170 	unsigned long curr = (unsigned long)current;
142171 	unsigned long zero = 0UL;
143172 
144
-	if (atomic_long_try_cmpxchg_acquire(&lock->owner, &zero, curr))
173
+	if (atomic_long_try_cmpxchg_acquire(&lock->owner, &zero, curr)) {
174
+		trace_android_vh_record_mutex_lock_starttime(current, jiffies);
145175 		return true;
176
+	}
146177 
147178 	return false;
148179 }
....@@ -181,9 +212,12 @@
181212 __mutex_add_waiter(struct mutex *lock, struct mutex_waiter *waiter,
182213 		   struct list_head *list)
183214 {
215
+	bool already_on_list = false;
184216 	debug_mutex_add_waiter(lock, waiter, current);
185217 
186
-	list_add_tail(&waiter->list, list);
218
+	trace_android_vh_alter_mutex_list_add(lock, waiter, list, &already_on_list);
219
+	if (!already_on_list)
220
+		list_add_tail(&waiter->list, list);
187221 	if (__mutex_waiter_is_first(lock, waiter))
188222 		__mutex_set_flag(lock, MUTEX_FLAG_WAITERS);
189223 }
....@@ -716,6 +750,7 @@
716750  */
717751 void __sched mutex_unlock(struct mutex *lock)
718752 {
753
+	trace_android_vh_record_mutex_lock_starttime(current, 0);
719754 #ifndef CONFIG_DEBUG_LOCK_ALLOC
720755 	if (__mutex_unlock_fast(lock))
721756 		return;
....@@ -919,6 +954,10 @@
919954 
920955 	might_sleep();
921956 
957
+#ifdef CONFIG_DEBUG_MUTEXES
958
+	DEBUG_LOCKS_WARN_ON(lock->magic != lock);
959
+#endif
960
+
922961 	ww = container_of(lock, struct ww_mutex, base);
923962 	if (ww_ctx) {
924963 		if (unlikely(ww_ctx == READ_ONCE(ww->ctx)))
....@@ -942,6 +981,7 @@
942981 		lock_acquired(&lock->dep_map, ip);
943982 		if (ww_ctx)
944983 			ww_mutex_set_context_fastpath(ww, ww_ctx);
984
+		trace_android_vh_record_mutex_lock_starttime(current, jiffies);
945985 		preempt_enable();
946986 		return 0;
947987 	}
....@@ -983,6 +1023,7 @@
9831023 
9841024 	waiter.task = current;
9851025 
1026
+	trace_android_vh_mutex_wait_start(lock);
9861027 	set_current_state(state);
9871028 	for (;;) {
9881029 		bool first;
....@@ -1001,7 +1042,7 @@
10011042 		 * wait_lock. This ensures the lock cancellation is ordered
10021043 		 * against mutex_unlock() and wake-ups do not go missing.
10031044 		 */
1004
-		if (unlikely(signal_pending_state(state, current))) {
1045
+		if (signal_pending_state(state, current)) {
10051046 			ret = -EINTR;
10061047 			goto err;
10071048 		}
....@@ -1034,6 +1075,7 @@
10341075 	spin_lock(&lock->wait_lock);
10351076 acquired:
10361077 	__set_current_state(TASK_RUNNING);
1078
+	trace_android_vh_mutex_wait_finish(lock);
10371079 
10381080 	if (ww_ctx) {
10391081 		/*
....@@ -1058,15 +1100,17 @@
10581100 
10591101 	spin_unlock(&lock->wait_lock);
10601102 	preempt_enable();
1103
+	trace_android_vh_record_mutex_lock_starttime(current, jiffies);
10611104 	return 0;
10621105 
10631106 err:
10641107 	__set_current_state(TASK_RUNNING);
1108
+	trace_android_vh_mutex_wait_finish(lock);
10651109 	__mutex_remove_waiter(lock, &waiter);
10661110 err_early_kill:
10671111 	spin_unlock(&lock->wait_lock);
10681112 	debug_mutex_free_waiter(&waiter);
1069
-	mutex_release(&lock->dep_map, 1, ip);
1113
+	mutex_release(&lock->dep_map, ip);
10701114 	preempt_enable();
10711115 	return ret;
10721116 }
....@@ -1200,7 +1244,7 @@
12001244 	DEFINE_WAKE_Q(wake_q);
12011245 	unsigned long owner;
12021246 
1203
-	mutex_release(&lock->dep_map, 1, ip);
1247
+	mutex_release(&lock->dep_map, ip);
12041248 
12051249 	/*
12061250 	 * Release the lock before (potentially) taking the spinlock such that
....@@ -1250,9 +1294,11 @@
12501294 	if (owner & MUTEX_FLAG_HANDOFF)
12511295 		__mutex_handoff(lock, next);
12521296 
1297
+	trace_android_vh_mutex_unlock_slowpath(lock);
12531298 	spin_unlock(&lock->wait_lock);
12541299 
12551300 	wake_up_q(&wake_q);
1301
+	trace_android_vh_mutex_unlock_slowpath_end(lock, next);
12561302 }
12571303 
12581304 #ifndef CONFIG_DEBUG_LOCK_ALLOC
....@@ -1384,10 +1430,17 @@
13841430  */
13851431 int __sched mutex_trylock(struct mutex *lock)
13861432 {
1387
-	bool locked = __mutex_trylock(lock);
1433
+	bool locked;
13881434 
1389
-	if (locked)
1435
+#ifdef CONFIG_DEBUG_MUTEXES
1436
+	DEBUG_LOCKS_WARN_ON(lock->magic != lock);
1437
+#endif
1438
+
1439
+	locked = __mutex_trylock(lock);
1440
+	if (locked) {
1441
+		trace_android_vh_record_mutex_lock_starttime(current, jiffies);
13901442 		mutex_acquire(&lock->dep_map, 0, 1, _RET_IP_);
1443
+	}
13911444 
13921445 	return locked;
13931446 }