.. | .. |
---|
1 | | -/* |
---|
2 | | - */ |
---|
| 1 | +// SPDX-License-Identifier: GPL-2.0-only |
---|
3 | 2 | #include <linux/sched/debug.h> |
---|
4 | 3 | #include <linux/export.h> |
---|
5 | 4 | |
---|
.. | .. |
---|
62 | 61 | lock->rtmutex.save_state = 1; |
---|
63 | 62 | } |
---|
64 | 63 | |
---|
65 | | -int __read_rt_trylock(struct rt_rw_lock *lock) |
---|
| 64 | +static int __read_rt_trylock(struct rt_rw_lock *lock) |
---|
66 | 65 | { |
---|
67 | 66 | int r, old; |
---|
68 | 67 | |
---|
.. | .. |
---|
79 | 78 | return 0; |
---|
80 | 79 | } |
---|
81 | 80 | |
---|
82 | | -void __sched __read_rt_lock(struct rt_rw_lock *lock) |
---|
| 81 | +static void __read_rt_lock(struct rt_rw_lock *lock) |
---|
83 | 82 | { |
---|
84 | 83 | struct rt_mutex *m = &lock->rtmutex; |
---|
85 | 84 | struct rt_mutex_waiter waiter; |
---|
.. | .. |
---|
142 | 141 | debug_rt_mutex_free_waiter(&waiter); |
---|
143 | 142 | } |
---|
144 | 143 | |
---|
145 | | -void __read_rt_unlock(struct rt_rw_lock *lock) |
---|
| 144 | +static void __read_rt_unlock(struct rt_rw_lock *lock) |
---|
146 | 145 | { |
---|
147 | 146 | struct rt_mutex *m = &lock->rtmutex; |
---|
148 | 147 | struct task_struct *tsk; |
---|
.. | .. |
---|
178 | 177 | rt_spin_lock_slowunlock(m); |
---|
179 | 178 | } |
---|
180 | 179 | |
---|
181 | | -void __sched __write_rt_lock(struct rt_rw_lock *lock) |
---|
| 180 | +static void __write_rt_lock(struct rt_rw_lock *lock) |
---|
182 | 181 | { |
---|
183 | 182 | struct rt_mutex *m = &lock->rtmutex; |
---|
184 | 183 | struct task_struct *self = current; |
---|
.. | .. |
---|
212 | 211 | raw_spin_unlock_irqrestore(&m->wait_lock, flags); |
---|
213 | 212 | |
---|
214 | 213 | if (atomic_read(&lock->readers) != 0) |
---|
215 | | - schedule(); |
---|
| 214 | + preempt_schedule_lock(); |
---|
216 | 215 | |
---|
217 | 216 | raw_spin_lock_irqsave(&m->wait_lock, flags); |
---|
218 | 217 | |
---|
.. | .. |
---|
222 | 221 | } |
---|
223 | 222 | } |
---|
224 | 223 | |
---|
225 | | -int __write_rt_trylock(struct rt_rw_lock *lock) |
---|
| 224 | +static int __write_rt_trylock(struct rt_rw_lock *lock) |
---|
226 | 225 | { |
---|
227 | 226 | struct rt_mutex *m = &lock->rtmutex; |
---|
228 | 227 | unsigned long flags; |
---|
.. | .. |
---|
242 | 241 | return 0; |
---|
243 | 242 | } |
---|
244 | 243 | |
---|
245 | | -void __write_rt_unlock(struct rt_rw_lock *lock) |
---|
| 244 | +static void __write_rt_unlock(struct rt_rw_lock *lock) |
---|
246 | 245 | { |
---|
247 | 246 | struct rt_mutex *m = &lock->rtmutex; |
---|
248 | 247 | unsigned long flags; |
---|
249 | 248 | |
---|
250 | 249 | raw_spin_lock_irqsave(&m->wait_lock, flags); |
---|
251 | 250 | __write_unlock_common(lock, WRITER_BIAS, flags); |
---|
252 | | -} |
---|
253 | | - |
---|
254 | | -/* Map the reader biased implementation */ |
---|
255 | | -static inline int do_read_rt_trylock(rwlock_t *rwlock) |
---|
256 | | -{ |
---|
257 | | - return __read_rt_trylock(rwlock); |
---|
258 | | -} |
---|
259 | | - |
---|
260 | | -static inline int do_write_rt_trylock(rwlock_t *rwlock) |
---|
261 | | -{ |
---|
262 | | - return __write_rt_trylock(rwlock); |
---|
263 | | -} |
---|
264 | | - |
---|
265 | | -static inline void do_read_rt_lock(rwlock_t *rwlock) |
---|
266 | | -{ |
---|
267 | | - __read_rt_lock(rwlock); |
---|
268 | | -} |
---|
269 | | - |
---|
270 | | -static inline void do_write_rt_lock(rwlock_t *rwlock) |
---|
271 | | -{ |
---|
272 | | - __write_rt_lock(rwlock); |
---|
273 | | -} |
---|
274 | | - |
---|
275 | | -static inline void do_read_rt_unlock(rwlock_t *rwlock) |
---|
276 | | -{ |
---|
277 | | - __read_rt_unlock(rwlock); |
---|
278 | | -} |
---|
279 | | - |
---|
280 | | -static inline void do_write_rt_unlock(rwlock_t *rwlock) |
---|
281 | | -{ |
---|
282 | | - __write_rt_unlock(rwlock); |
---|
283 | | -} |
---|
284 | | - |
---|
285 | | -static inline void do_rwlock_rt_init(rwlock_t *rwlock, const char *name, |
---|
286 | | - struct lock_class_key *key) |
---|
287 | | -{ |
---|
288 | | - __rwlock_biased_rt_init(rwlock, name, key); |
---|
289 | 251 | } |
---|
290 | 252 | |
---|
291 | 253 | int __lockfunc rt_read_can_lock(rwlock_t *rwlock) |
---|
.. | .. |
---|
305 | 267 | { |
---|
306 | 268 | int ret; |
---|
307 | 269 | |
---|
308 | | - sleeping_lock_inc(); |
---|
309 | | - migrate_disable(); |
---|
310 | | - ret = do_read_rt_trylock(rwlock); |
---|
| 270 | + ret = __read_rt_trylock(rwlock); |
---|
311 | 271 | if (ret) { |
---|
312 | 272 | rwlock_acquire_read(&rwlock->dep_map, 0, 1, _RET_IP_); |
---|
313 | 273 | rcu_read_lock(); |
---|
314 | | - } else { |
---|
315 | | - migrate_enable(); |
---|
316 | | - sleeping_lock_dec(); |
---|
| 274 | + migrate_disable(); |
---|
317 | 275 | } |
---|
318 | 276 | return ret; |
---|
319 | 277 | } |
---|
.. | .. |
---|
323 | 281 | { |
---|
324 | 282 | int ret; |
---|
325 | 283 | |
---|
326 | | - sleeping_lock_inc(); |
---|
327 | | - migrate_disable(); |
---|
328 | | - ret = do_write_rt_trylock(rwlock); |
---|
| 284 | + ret = __write_rt_trylock(rwlock); |
---|
329 | 285 | if (ret) { |
---|
330 | 286 | rwlock_acquire(&rwlock->dep_map, 0, 1, _RET_IP_); |
---|
331 | 287 | rcu_read_lock(); |
---|
332 | | - } else { |
---|
333 | | - migrate_enable(); |
---|
334 | | - sleeping_lock_dec(); |
---|
| 288 | + migrate_disable(); |
---|
335 | 289 | } |
---|
336 | 290 | return ret; |
---|
337 | 291 | } |
---|
.. | .. |
---|
339 | 293 | |
---|
340 | 294 | void __lockfunc rt_read_lock(rwlock_t *rwlock) |
---|
341 | 295 | { |
---|
342 | | - sleeping_lock_inc(); |
---|
| 296 | + rwlock_acquire_read(&rwlock->dep_map, 0, 0, _RET_IP_); |
---|
| 297 | + __read_rt_lock(rwlock); |
---|
343 | 298 | rcu_read_lock(); |
---|
344 | 299 | migrate_disable(); |
---|
345 | | - rwlock_acquire_read(&rwlock->dep_map, 0, 0, _RET_IP_); |
---|
346 | | - do_read_rt_lock(rwlock); |
---|
347 | 300 | } |
---|
348 | 301 | EXPORT_SYMBOL(rt_read_lock); |
---|
349 | 302 | |
---|
350 | 303 | void __lockfunc rt_write_lock(rwlock_t *rwlock) |
---|
351 | 304 | { |
---|
352 | | - sleeping_lock_inc(); |
---|
| 305 | + rwlock_acquire(&rwlock->dep_map, 0, 0, _RET_IP_); |
---|
| 306 | + __write_rt_lock(rwlock); |
---|
353 | 307 | rcu_read_lock(); |
---|
354 | 308 | migrate_disable(); |
---|
355 | | - rwlock_acquire(&rwlock->dep_map, 0, 0, _RET_IP_); |
---|
356 | | - do_write_rt_lock(rwlock); |
---|
357 | 309 | } |
---|
358 | 310 | EXPORT_SYMBOL(rt_write_lock); |
---|
359 | 311 | |
---|
360 | 312 | void __lockfunc rt_read_unlock(rwlock_t *rwlock) |
---|
361 | 313 | { |
---|
362 | | - rwlock_release(&rwlock->dep_map, 1, _RET_IP_); |
---|
363 | | - do_read_rt_unlock(rwlock); |
---|
| 314 | + rwlock_release(&rwlock->dep_map, _RET_IP_); |
---|
364 | 315 | migrate_enable(); |
---|
365 | 316 | rcu_read_unlock(); |
---|
366 | | - sleeping_lock_dec(); |
---|
| 317 | + __read_rt_unlock(rwlock); |
---|
367 | 318 | } |
---|
368 | 319 | EXPORT_SYMBOL(rt_read_unlock); |
---|
369 | 320 | |
---|
370 | 321 | void __lockfunc rt_write_unlock(rwlock_t *rwlock) |
---|
371 | 322 | { |
---|
372 | | - rwlock_release(&rwlock->dep_map, 1, _RET_IP_); |
---|
373 | | - do_write_rt_unlock(rwlock); |
---|
| 323 | + rwlock_release(&rwlock->dep_map, _RET_IP_); |
---|
374 | 324 | migrate_enable(); |
---|
375 | 325 | rcu_read_unlock(); |
---|
376 | | - sleeping_lock_dec(); |
---|
| 326 | + __write_rt_unlock(rwlock); |
---|
377 | 327 | } |
---|
378 | 328 | EXPORT_SYMBOL(rt_write_unlock); |
---|
379 | 329 | |
---|
380 | 330 | void __rt_rwlock_init(rwlock_t *rwlock, char *name, struct lock_class_key *key) |
---|
381 | 331 | { |
---|
382 | | - do_rwlock_rt_init(rwlock, name, key); |
---|
| 332 | + __rwlock_biased_rt_init(rwlock, name, key); |
---|
383 | 333 | } |
---|
384 | 334 | EXPORT_SYMBOL(__rt_rwlock_init); |
---|