.. | .. |
---|
| 1 | +// SPDX-License-Identifier: GPL-2.0-only |
---|
1 | 2 | /* |
---|
2 | | - * kernel/rt.c |
---|
3 | | - * |
---|
4 | 3 | * Real-Time Preemption Support |
---|
5 | 4 | * |
---|
6 | 5 | * started by Ingo Molnar: |
---|
.. | .. |
---|
66 | 65 | #include <linux/fs.h> |
---|
67 | 66 | #include <linux/futex.h> |
---|
68 | 67 | #include <linux/hrtimer.h> |
---|
| 68 | +#include <linux/blkdev.h> |
---|
69 | 69 | |
---|
70 | 70 | #include "rtmutex_common.h" |
---|
71 | 71 | |
---|
.. | .. |
---|
86 | 86 | } |
---|
87 | 87 | EXPORT_SYMBOL(__mutex_do_init); |
---|
88 | 88 | |
---|
| 89 | +static int _mutex_lock_blk_flush(struct mutex *lock, int state) |
---|
| 90 | +{ |
---|
| 91 | + /* |
---|
| 92 | + * Flush blk before ->pi_blocked_on is set. At schedule() time it is too |
---|
| 93 | + * late if one of the callbacks needs to acquire a sleeping lock. |
---|
| 94 | + */ |
---|
| 95 | + if (blk_needs_flush_plug(current)) |
---|
| 96 | + blk_schedule_flush_plug(current); |
---|
| 97 | + return __rt_mutex_lock_state(&lock->lock, state); |
---|
| 98 | +} |
---|
| 99 | + |
---|
89 | 100 | void __lockfunc _mutex_lock(struct mutex *lock) |
---|
90 | 101 | { |
---|
91 | 102 | mutex_acquire(&lock->dep_map, 0, 0, _RET_IP_); |
---|
92 | | - __rt_mutex_lock_state(&lock->lock, TASK_UNINTERRUPTIBLE); |
---|
| 103 | + _mutex_lock_blk_flush(lock, TASK_UNINTERRUPTIBLE); |
---|
93 | 104 | } |
---|
94 | 105 | EXPORT_SYMBOL(_mutex_lock); |
---|
95 | | - |
---|
96 | | -void __lockfunc _mutex_lock_io(struct mutex *lock) |
---|
97 | | -{ |
---|
98 | | - int token; |
---|
99 | | - |
---|
100 | | - token = io_schedule_prepare(); |
---|
101 | | - _mutex_lock(lock); |
---|
102 | | - io_schedule_finish(token); |
---|
103 | | -} |
---|
104 | | -EXPORT_SYMBOL_GPL(_mutex_lock_io); |
---|
105 | | - |
---|
106 | | -int __lockfunc _mutex_lock_interruptible(struct mutex *lock) |
---|
107 | | -{ |
---|
108 | | - int ret; |
---|
109 | | - |
---|
110 | | - mutex_acquire(&lock->dep_map, 0, 0, _RET_IP_); |
---|
111 | | - ret = __rt_mutex_lock_state(&lock->lock, TASK_INTERRUPTIBLE); |
---|
112 | | - if (ret) |
---|
113 | | - mutex_release(&lock->dep_map, 1, _RET_IP_); |
---|
114 | | - return ret; |
---|
115 | | -} |
---|
116 | | -EXPORT_SYMBOL(_mutex_lock_interruptible); |
---|
117 | | - |
---|
118 | | -int __lockfunc _mutex_lock_killable(struct mutex *lock) |
---|
119 | | -{ |
---|
120 | | - int ret; |
---|
121 | | - |
---|
122 | | - mutex_acquire(&lock->dep_map, 0, 0, _RET_IP_); |
---|
123 | | - ret = __rt_mutex_lock_state(&lock->lock, TASK_KILLABLE); |
---|
124 | | - if (ret) |
---|
125 | | - mutex_release(&lock->dep_map, 1, _RET_IP_); |
---|
126 | | - return ret; |
---|
127 | | -} |
---|
128 | | -EXPORT_SYMBOL(_mutex_lock_killable); |
---|
129 | | - |
---|
130 | | -#ifdef CONFIG_DEBUG_LOCK_ALLOC |
---|
131 | | -void __lockfunc _mutex_lock_nested(struct mutex *lock, int subclass) |
---|
132 | | -{ |
---|
133 | | - mutex_acquire_nest(&lock->dep_map, subclass, 0, NULL, _RET_IP_); |
---|
134 | | - __rt_mutex_lock_state(&lock->lock, TASK_UNINTERRUPTIBLE); |
---|
135 | | -} |
---|
136 | | -EXPORT_SYMBOL(_mutex_lock_nested); |
---|
137 | 106 | |
---|
138 | 107 | void __lockfunc _mutex_lock_io_nested(struct mutex *lock, int subclass) |
---|
139 | 108 | { |
---|
.. | .. |
---|
148 | 117 | } |
---|
149 | 118 | EXPORT_SYMBOL_GPL(_mutex_lock_io_nested); |
---|
150 | 119 | |
---|
| 120 | +int __lockfunc _mutex_lock_interruptible(struct mutex *lock) |
---|
| 121 | +{ |
---|
| 122 | + int ret; |
---|
| 123 | + |
---|
| 124 | + mutex_acquire(&lock->dep_map, 0, 0, _RET_IP_); |
---|
| 125 | + ret = _mutex_lock_blk_flush(lock, TASK_INTERRUPTIBLE); |
---|
| 126 | + if (ret) |
---|
| 127 | + mutex_release(&lock->dep_map, _RET_IP_); |
---|
| 128 | + return ret; |
---|
| 129 | +} |
---|
| 130 | +EXPORT_SYMBOL(_mutex_lock_interruptible); |
---|
| 131 | + |
---|
| 132 | +int __lockfunc _mutex_lock_killable(struct mutex *lock) |
---|
| 133 | +{ |
---|
| 134 | + int ret; |
---|
| 135 | + |
---|
| 136 | + mutex_acquire(&lock->dep_map, 0, 0, _RET_IP_); |
---|
| 137 | + ret = _mutex_lock_blk_flush(lock, TASK_KILLABLE); |
---|
| 138 | + if (ret) |
---|
| 139 | + mutex_release(&lock->dep_map, _RET_IP_); |
---|
| 140 | + return ret; |
---|
| 141 | +} |
---|
| 142 | +EXPORT_SYMBOL(_mutex_lock_killable); |
---|
| 143 | + |
---|
| 144 | +#ifdef CONFIG_DEBUG_LOCK_ALLOC |
---|
| 145 | +void __lockfunc _mutex_lock_nested(struct mutex *lock, int subclass) |
---|
| 146 | +{ |
---|
| 147 | + mutex_acquire_nest(&lock->dep_map, subclass, 0, NULL, _RET_IP_); |
---|
| 148 | + _mutex_lock_blk_flush(lock, TASK_UNINTERRUPTIBLE); |
---|
| 149 | +} |
---|
| 150 | +EXPORT_SYMBOL(_mutex_lock_nested); |
---|
| 151 | + |
---|
151 | 152 | void __lockfunc _mutex_lock_nest_lock(struct mutex *lock, struct lockdep_map *nest) |
---|
152 | 153 | { |
---|
153 | 154 | mutex_acquire_nest(&lock->dep_map, 0, 0, nest, _RET_IP_); |
---|
154 | | - __rt_mutex_lock_state(&lock->lock, TASK_UNINTERRUPTIBLE); |
---|
| 155 | + _mutex_lock_blk_flush(lock, TASK_UNINTERRUPTIBLE); |
---|
155 | 156 | } |
---|
156 | 157 | EXPORT_SYMBOL(_mutex_lock_nest_lock); |
---|
157 | 158 | |
---|
.. | .. |
---|
160 | 161 | int ret; |
---|
161 | 162 | |
---|
162 | 163 | mutex_acquire_nest(&lock->dep_map, subclass, 0, NULL, _RET_IP_); |
---|
163 | | - ret = __rt_mutex_lock_state(&lock->lock, TASK_INTERRUPTIBLE); |
---|
| 164 | + ret = _mutex_lock_blk_flush(lock, TASK_INTERRUPTIBLE); |
---|
164 | 165 | if (ret) |
---|
165 | | - mutex_release(&lock->dep_map, 1, _RET_IP_); |
---|
| 166 | + mutex_release(&lock->dep_map, _RET_IP_); |
---|
166 | 167 | return ret; |
---|
167 | 168 | } |
---|
168 | 169 | EXPORT_SYMBOL(_mutex_lock_interruptible_nested); |
---|
.. | .. |
---|
172 | 173 | int ret; |
---|
173 | 174 | |
---|
174 | 175 | mutex_acquire(&lock->dep_map, subclass, 0, _RET_IP_); |
---|
175 | | - ret = __rt_mutex_lock_state(&lock->lock, TASK_KILLABLE); |
---|
| 176 | + ret = _mutex_lock_blk_flush(lock, TASK_KILLABLE); |
---|
176 | 177 | if (ret) |
---|
177 | | - mutex_release(&lock->dep_map, 1, _RET_IP_); |
---|
| 178 | + mutex_release(&lock->dep_map, _RET_IP_); |
---|
178 | 179 | return ret; |
---|
179 | 180 | } |
---|
180 | 181 | EXPORT_SYMBOL(_mutex_lock_killable_nested); |
---|
.. | .. |
---|
193 | 194 | |
---|
194 | 195 | void __lockfunc _mutex_unlock(struct mutex *lock) |
---|
195 | 196 | { |
---|
196 | | - mutex_release(&lock->dep_map, 1, _RET_IP_); |
---|
| 197 | + mutex_release(&lock->dep_map, _RET_IP_); |
---|
197 | 198 | __rt_mutex_unlock(&lock->lock); |
---|
198 | 199 | } |
---|
199 | 200 | EXPORT_SYMBOL(_mutex_unlock); |
---|