change wifi driver to cypress

hc

2024-01-05 071106ecf68c401173c58808b1cf5f68cc50d390

 kernel/fs/btrfs/locking.c
..
..
@@ -8,209 +8,398 @@
8
8
 #include <linux/spinlock.h>
9
9
 #include <linux/page-flags.h>
10
10
 #include <asm/bug.h>
11
+#include "misc.h"
11
12
 #include "ctree.h"
12
13
 #include "extent_io.h"
13
14
 #include "locking.h"
14
15
 
15
-static void btrfs_assert_tree_read_locked(struct extent_buffer *eb);
16
-
17
16
 /*
18
- * if we currently have a spinning reader or writer lock
19
- * (indicated by the rw flag) this will bump the count
20
- * of blocking holders and drop the spinlock.
17
+ * Extent buffer locking
18
+ * =====================
19
+ *
20
+ * The locks use a custom scheme that allows to do more operations than are
21
+ * available fromt current locking primitives. The building blocks are still
22
+ * rwlock and wait queues.
23
+ *
24
+ * Required semantics:
25
+ *
26
+ * - reader/writer exclusion
27
+ * - writer/writer exclusion
28
+ * - reader/reader sharing
29
+ * - spinning lock semantics
30
+ * - blocking lock semantics
31
+ * - try-lock semantics for readers and writers
32
+ * - one level nesting, allowing read lock to be taken by the same thread that
33
+ *   already has write lock
34
+ *
35
+ * The extent buffer locks (also called tree locks) manage access to eb data
36
+ * related to the storage in the b-tree (keys, items, but not the individual
37
+ * members of eb).
38
+ * We want concurrency of many readers and safe updates. The underlying locking
39
+ * is done by read-write spinlock and the blocking part is implemented using
40
+ * counters and wait queues.
41
+ *
42
+ * spinning semantics - the low-level rwlock is held so all other threads that
43
+ *                      want to take it are spinning on it.
44
+ *
45
+ * blocking semantics - the low-level rwlock is not held but the counter
46
+ *                      denotes how many times the blocking lock was held;
47
+ *                      sleeping is possible
48
+ *
49
+ * Write lock always allows only one thread to access the data.
50
+ *
51
+ *
52
+ * Debugging
53
+ * ---------
54
+ *
55
+ * There are additional state counters that are asserted in various contexts,
56
+ * removed from non-debug build to reduce extent_buffer size and for
57
+ * performance reasons.
58
+ *
59
+ *
60
+ * Lock recursion
61
+ * --------------
62
+ *
63
+ * A write operation on a tree might indirectly start a look up on the same
64
+ * tree.  This can happen when btrfs_cow_block locks the tree and needs to
65
+ * lookup free extents.
66
+ *
67
+ * btrfs_cow_block
68
+ *   ..
69
+ *   alloc_tree_block_no_bg_flush
70
+ *     btrfs_alloc_tree_block
71
+ *       btrfs_reserve_extent
72
+ *         ..
73
+ *         load_free_space_cache
74
+ *           ..
75
+ *           btrfs_lookup_file_extent
76
+ *             btrfs_search_slot
77
+ *
78
+ *
79
+ * Locking pattern - spinning
80
+ * --------------------------
81
+ *
82
+ * The simple locking scenario, the +--+ denotes the spinning section.
83
+ *
84
+ * +- btrfs_tree_lock
85
+ * | - extent_buffer::rwlock is held
86
+ * | - no heavy operations should happen, eg. IO, memory allocations, large
87
+ * |   structure traversals
88
+ * +- btrfs_tree_unock
89
+*
90
+*
91
+ * Locking pattern - blocking
92
+ * --------------------------
93
+ *
94
+ * The blocking write uses the following scheme.  The +--+ denotes the spinning
95
+ * section.
96
+ *
97
+ * +- btrfs_tree_lock
98
+ * |
99
+ * +- btrfs_set_lock_blocking_write
100
+ *
101
+ *   - allowed: IO, memory allocations, etc.
102
+ *
103
+ * -- btrfs_tree_unlock - note, no explicit unblocking necessary
104
+ *
105
+ *
106
+ * Blocking read is similar.
107
+ *
108
+ * +- btrfs_tree_read_lock
109
+ * |
110
+ * +- btrfs_set_lock_blocking_read
111
+ *
112
+ *  - heavy operations allowed
113
+ *
114
+ * +- btrfs_tree_read_unlock_blocking
115
+ * |
116
+ * +- btrfs_tree_read_unlock
117
+ *
21
118
  */
22
-void btrfs_set_lock_blocking_rw(struct extent_buffer *eb, int rw)
119
+
120
+#ifdef CONFIG_BTRFS_DEBUG
121
+static inline void btrfs_assert_spinning_writers_get(struct extent_buffer *eb)
23
122
 {
24
-	/*
25
-	 * no lock is required.  The lock owner may change if
26
-	 * we have a read lock, but it won't change to or away
27
-	 * from us.  If we have the write lock, we are the owner
28
-	 * and it'll never change.
29
-	 */
30
-	if (eb->lock_nested && current->pid == eb->lock_owner)
31
-		return;
32
-	if (rw == BTRFS_WRITE_LOCK) {
33
-		if (atomic_read(&eb->blocking_writers) == 0) {
34
-			WARN_ON(atomic_read(&eb->spinning_writers) != 1);
35
-			atomic_dec(&eb->spinning_writers);
36
-			btrfs_assert_tree_locked(eb);
37
-			atomic_inc(&eb->blocking_writers);
38
-			write_unlock(&eb->lock);
39
-		}
40
-	} else if (rw == BTRFS_READ_LOCK) {
41
-		btrfs_assert_tree_read_locked(eb);
42
-		atomic_inc(&eb->blocking_readers);
43
-		WARN_ON(atomic_read(&eb->spinning_readers) == 0);
44
-		atomic_dec(&eb->spinning_readers);
45
-		read_unlock(&eb->lock);
46
-	}
123
+	WARN_ON(eb->spinning_writers);
124
+	eb->spinning_writers++;
47
125
 }
48
126
 
49
-/*
50
- * if we currently have a blocking lock, take the spinlock
51
- * and drop our blocking count
52
- */
53
-void btrfs_clear_lock_blocking_rw(struct extent_buffer *eb, int rw)
127
+static inline void btrfs_assert_spinning_writers_put(struct extent_buffer *eb)
54
128
 {
55
-	/*
56
-	 * no lock is required.  The lock owner may change if
57
-	 * we have a read lock, but it won't change to or away
58
-	 * from us.  If we have the write lock, we are the owner
59
-	 * and it'll never change.
60
-	 */
61
-	if (eb->lock_nested && current->pid == eb->lock_owner)
62
-		return;
63
-
64
-	if (rw == BTRFS_WRITE_LOCK_BLOCKING) {
65
-		BUG_ON(atomic_read(&eb->blocking_writers) != 1);
66
-		write_lock(&eb->lock);
67
-		WARN_ON(atomic_read(&eb->spinning_writers));
68
-		atomic_inc(&eb->spinning_writers);
69
-		/* atomic_dec_and_test implies a barrier */
70
-		if (atomic_dec_and_test(&eb->blocking_writers))
71
-			cond_wake_up_nomb(&eb->write_lock_wq);
72
-	} else if (rw == BTRFS_READ_LOCK_BLOCKING) {
73
-		BUG_ON(atomic_read(&eb->blocking_readers) == 0);
74
-		read_lock(&eb->lock);
75
-		atomic_inc(&eb->spinning_readers);
76
-		/* atomic_dec_and_test implies a barrier */
77
-		if (atomic_dec_and_test(&eb->blocking_readers))
78
-			cond_wake_up_nomb(&eb->read_lock_wq);
79
-	}
129
+	WARN_ON(eb->spinning_writers != 1);
130
+	eb->spinning_writers--;
80
131
 }
81
132
 
82
-/*
83
- * take a spinning read lock.  This will wait for any blocking
84
- * writers
85
- */
86
-void btrfs_tree_read_lock(struct extent_buffer *eb)
133
+static inline void btrfs_assert_no_spinning_writers(struct extent_buffer *eb)
87
134
 {
88
-again:
89
-	BUG_ON(!atomic_read(&eb->blocking_writers) &&
90
-	       current->pid == eb->lock_owner);
135
+	WARN_ON(eb->spinning_writers);
136
+}
91
137
 
92
-	read_lock(&eb->lock);
93
-	if (atomic_read(&eb->blocking_writers) &&
94
-	    current->pid == eb->lock_owner) {
95
-		/*
96
-		 * This extent is already write-locked by our thread. We allow
97
-		 * an additional read lock to be added because it's for the same
98
-		 * thread. btrfs_find_all_roots() depends on this as it may be
99
-		 * called on a partly (write-)locked tree.
100
-		 */
101
-		BUG_ON(eb->lock_nested);
102
-		eb->lock_nested = 1;
103
-		read_unlock(&eb->lock);
104
-		return;
105
-	}
106
-	if (atomic_read(&eb->blocking_writers)) {
107
-		read_unlock(&eb->lock);
108
-		wait_event(eb->write_lock_wq,
109
-			   atomic_read(&eb->blocking_writers) == 0);
110
-		goto again;
111
-	}
112
-	atomic_inc(&eb->read_locks);
138
+static inline void btrfs_assert_spinning_readers_get(struct extent_buffer *eb)
139
+{
113
140
 	atomic_inc(&eb->spinning_readers);
114
141
 }
115
142
 
143
+static inline void btrfs_assert_spinning_readers_put(struct extent_buffer *eb)
144
+{
145
+	WARN_ON(atomic_read(&eb->spinning_readers) == 0);
146
+	atomic_dec(&eb->spinning_readers);
147
+}
148
+
149
+static inline void btrfs_assert_tree_read_locks_get(struct extent_buffer *eb)
150
+{
151
+	atomic_inc(&eb->read_locks);
152
+}
153
+
154
+static inline void btrfs_assert_tree_read_locks_put(struct extent_buffer *eb)
155
+{
156
+	atomic_dec(&eb->read_locks);
157
+}
158
+
159
+static inline void btrfs_assert_tree_read_locked(struct extent_buffer *eb)
160
+{
161
+	BUG_ON(!atomic_read(&eb->read_locks));
162
+}
163
+
164
+static inline void btrfs_assert_tree_write_locks_get(struct extent_buffer *eb)
165
+{
166
+	eb->write_locks++;
167
+}
168
+
169
+static inline void btrfs_assert_tree_write_locks_put(struct extent_buffer *eb)
170
+{
171
+	eb->write_locks--;
172
+}
173
+
174
+#else
175
+static void btrfs_assert_spinning_writers_get(struct extent_buffer *eb) { }
176
+static void btrfs_assert_spinning_writers_put(struct extent_buffer *eb) { }
177
+static void btrfs_assert_no_spinning_writers(struct extent_buffer *eb) { }
178
+static void btrfs_assert_spinning_readers_put(struct extent_buffer *eb) { }
179
+static void btrfs_assert_spinning_readers_get(struct extent_buffer *eb) { }
180
+static void btrfs_assert_tree_read_locked(struct extent_buffer *eb) { }
181
+static void btrfs_assert_tree_read_locks_get(struct extent_buffer *eb) { }
182
+static void btrfs_assert_tree_read_locks_put(struct extent_buffer *eb) { }
183
+static void btrfs_assert_tree_write_locks_get(struct extent_buffer *eb) { }
184
+static void btrfs_assert_tree_write_locks_put(struct extent_buffer *eb) { }
185
+#endif
186
+
116
187
 /*
117
- * take a spinning read lock.
118
- * returns 1 if we get the read lock and 0 if we don't
119
- * this won't wait for blocking writers
188
+ * Mark already held read lock as blocking. Can be nested in write lock by the
189
+ * same thread.
190
+ *
191
+ * Use when there are potentially long operations ahead so other thread waiting
192
+ * on the lock will not actively spin but sleep instead.
193
+ *
194
+ * The rwlock is released and blocking reader counter is increased.
195
+ */
196
+void btrfs_set_lock_blocking_read(struct extent_buffer *eb)
197
+{
198
+	trace_btrfs_set_lock_blocking_read(eb);
199
+	/*
200
+	 * No lock is required.  The lock owner may change if we have a read
201
+	 * lock, but it won't change to or away from us.  If we have the write
202
+	 * lock, we are the owner and it'll never change.
203
+	 */
204
+	if (eb->lock_recursed && current->pid == eb->lock_owner)
205
+		return;
206
+	btrfs_assert_tree_read_locked(eb);
207
+	atomic_inc(&eb->blocking_readers);
208
+	btrfs_assert_spinning_readers_put(eb);
209
+	read_unlock(&eb->lock);
210
+}
211
+
212
+/*
213
+ * Mark already held write lock as blocking.
214
+ *
215
+ * Use when there are potentially long operations ahead so other threads
216
+ * waiting on the lock will not actively spin but sleep instead.
217
+ *
218
+ * The rwlock is released and blocking writers is set.
219
+ */
220
+void btrfs_set_lock_blocking_write(struct extent_buffer *eb)
221
+{
222
+	trace_btrfs_set_lock_blocking_write(eb);
223
+	/*
224
+	 * No lock is required.  The lock owner may change if we have a read
225
+	 * lock, but it won't change to or away from us.  If we have the write
226
+	 * lock, we are the owner and it'll never change.
227
+	 */
228
+	if (eb->lock_recursed && current->pid == eb->lock_owner)
229
+		return;
230
+	if (eb->blocking_writers == 0) {
231
+		btrfs_assert_spinning_writers_put(eb);
232
+		btrfs_assert_tree_locked(eb);
233
+		WRITE_ONCE(eb->blocking_writers, 1);
234
+		write_unlock(&eb->lock);
235
+	}
236
+}
237
+
238
+/*
239
+ * Lock the extent buffer for read. Wait for any writers (spinning or blocking).
240
+ * Can be nested in write lock by the same thread.
241
+ *
242
+ * Use when the locked section does only lightweight actions and busy waiting
243
+ * would be cheaper than making other threads do the wait/wake loop.
244
+ *
245
+ * The rwlock is held upon exit.
246
+ */
247
+void __btrfs_tree_read_lock(struct extent_buffer *eb, enum btrfs_lock_nesting nest,
248
+			    bool recurse)
249
+{
250
+	u64 start_ns = 0;
251
+
252
+	if (trace_btrfs_tree_read_lock_enabled())
253
+		start_ns = ktime_get_ns();
254
+again:
255
+	read_lock(&eb->lock);
256
+	BUG_ON(eb->blocking_writers == 0 &&
257
+	       current->pid == eb->lock_owner);
258
+	if (eb->blocking_writers) {
259
+		if (current->pid == eb->lock_owner) {
260
+			/*
261
+			 * This extent is already write-locked by our thread.
262
+			 * We allow an additional read lock to be added because
263
+			 * it's for the same thread. btrfs_find_all_roots()
264
+			 * depends on this as it may be called on a partly
265
+			 * (write-)locked tree.
266
+			 */
267
+			WARN_ON(!recurse);
268
+			BUG_ON(eb->lock_recursed);
269
+			eb->lock_recursed = true;
270
+			read_unlock(&eb->lock);
271
+			trace_btrfs_tree_read_lock(eb, start_ns);
272
+			return;
273
+		}
274
+		read_unlock(&eb->lock);
275
+		wait_event(eb->write_lock_wq,
276
+			   READ_ONCE(eb->blocking_writers) == 0);
277
+		goto again;
278
+	}
279
+	btrfs_assert_tree_read_locks_get(eb);
280
+	btrfs_assert_spinning_readers_get(eb);
281
+	trace_btrfs_tree_read_lock(eb, start_ns);
282
+}
283
+
284
+void btrfs_tree_read_lock(struct extent_buffer *eb)
285
+{
286
+	__btrfs_tree_read_lock(eb, BTRFS_NESTING_NORMAL, false);
287
+}
288
+
289
+/*
290
+ * Lock extent buffer for read, optimistically expecting that there are no
291
+ * contending blocking writers. If there are, don't wait.
292
+ *
293
+ * Return 1 if the rwlock has been taken, 0 otherwise
120
294
  */
121
295
 int btrfs_tree_read_lock_atomic(struct extent_buffer *eb)
122
296
 {
123
-	if (atomic_read(&eb->blocking_writers))
297
+	if (READ_ONCE(eb->blocking_writers))
124
298
 		return 0;
125
299
 
126
300
 	read_lock(&eb->lock);
127
-	if (atomic_read(&eb->blocking_writers)) {
301
+	/* Refetch value after lock */
302
+	if (READ_ONCE(eb->blocking_writers)) {
128
303
 		read_unlock(&eb->lock);
129
304
 		return 0;
130
305
 	}
131
-	atomic_inc(&eb->read_locks);
132
-	atomic_inc(&eb->spinning_readers);
306
+	btrfs_assert_tree_read_locks_get(eb);
307
+	btrfs_assert_spinning_readers_get(eb);
308
+	trace_btrfs_tree_read_lock_atomic(eb);
133
309
 	return 1;
134
310
 }
135
311
 
136
312
 /*
137
- * returns 1 if we get the read lock and 0 if we don't
138
- * this won't wait for blocking writers
313
+ * Try-lock for read. Don't block or wait for contending writers.
314
+ *
315
+ * Retrun 1 if the rwlock has been taken, 0 otherwise
139
316
  */
140
317
 int btrfs_try_tree_read_lock(struct extent_buffer *eb)
141
318
 {
142
-	if (atomic_read(&eb->blocking_writers))
319
+	if (READ_ONCE(eb->blocking_writers))
143
320
 		return 0;
144
321
 
145
322
 	if (!read_trylock(&eb->lock))
146
323
 		return 0;
147
324
 
148
-	if (atomic_read(&eb->blocking_writers)) {
325
+	/* Refetch value after lock */
326
+	if (READ_ONCE(eb->blocking_writers)) {
149
327
 		read_unlock(&eb->lock);
150
328
 		return 0;
151
329
 	}
152
-	atomic_inc(&eb->read_locks);
153
-	atomic_inc(&eb->spinning_readers);
330
+	btrfs_assert_tree_read_locks_get(eb);
331
+	btrfs_assert_spinning_readers_get(eb);
332
+	trace_btrfs_try_tree_read_lock(eb);
154
333
 	return 1;
155
334
 }
156
335
 
157
336
 /*
158
- * returns 1 if we get the read lock and 0 if we don't
159
- * this won't wait for blocking writers or readers
337
+ * Try-lock for write. May block until the lock is uncontended, but does not
338
+ * wait until it is free.
339
+ *
340
+ * Retrun 1 if the rwlock has been taken, 0 otherwise
160
341
  */
161
342
 int btrfs_try_tree_write_lock(struct extent_buffer *eb)
162
343
 {
163
-	if (atomic_read(&eb->blocking_writers) ||
164
-	    atomic_read(&eb->blocking_readers))
344
+	if (READ_ONCE(eb->blocking_writers) || atomic_read(&eb->blocking_readers))
165
345
 		return 0;
166
346
 
167
347
 	write_lock(&eb->lock);
168
-	if (atomic_read(&eb->blocking_writers) ||
169
-	    atomic_read(&eb->blocking_readers)) {
348
+	/* Refetch value after lock */
349
+	if (READ_ONCE(eb->blocking_writers) || atomic_read(&eb->blocking_readers)) {
170
350
 		write_unlock(&eb->lock);
171
351
 		return 0;
172
352
 	}
173
-	atomic_inc(&eb->write_locks);
174
-	atomic_inc(&eb->spinning_writers);
353
+	btrfs_assert_tree_write_locks_get(eb);
354
+	btrfs_assert_spinning_writers_get(eb);
175
355
 	eb->lock_owner = current->pid;
356
+	trace_btrfs_try_tree_write_lock(eb);
176
357
 	return 1;
177
358
 }
178
359
 
179
360
 /*
180
- * drop a spinning read lock
361
+ * Release read lock. Must be used only if the lock is in spinning mode.  If
362
+ * the read lock is nested, must pair with read lock before the write unlock.
363
+ *
364
+ * The rwlock is not held upon exit.
181
365
  */
182
366
 void btrfs_tree_read_unlock(struct extent_buffer *eb)
183
367
 {
368
+	trace_btrfs_tree_read_unlock(eb);
184
369
 	/*
185
370
 	 * if we're nested, we have the write lock.  No new locking
186
371
 	 * is needed as long as we are the lock owner.
187
-	 * The write unlock will do a barrier for us, and the lock_nested
372
+	 * The write unlock will do a barrier for us, and the lock_recursed
188
373
 	 * field only matters to the lock owner.
189
374
 	 */
190
-	if (eb->lock_nested && current->pid == eb->lock_owner) {
191
-		eb->lock_nested = 0;
375
+	if (eb->lock_recursed && current->pid == eb->lock_owner) {
376
+		eb->lock_recursed = false;
192
377
 		return;
193
378
 	}
194
379
 	btrfs_assert_tree_read_locked(eb);
195
-	WARN_ON(atomic_read(&eb->spinning_readers) == 0);
196
-	atomic_dec(&eb->spinning_readers);
197
-	atomic_dec(&eb->read_locks);
380
+	btrfs_assert_spinning_readers_put(eb);
381
+	btrfs_assert_tree_read_locks_put(eb);
198
382
 	read_unlock(&eb->lock);
199
383
 }
200
384
 
201
385
 /*
202
- * drop a blocking read lock
386
+ * Release read lock, previously set to blocking by a pairing call to
387
+ * btrfs_set_lock_blocking_read(). Can be nested in write lock by the same
388
+ * thread.
389
+ *
390
+ * State of rwlock is unchanged, last reader wakes waiting threads.
203
391
  */
204
392
 void btrfs_tree_read_unlock_blocking(struct extent_buffer *eb)
205
393
 {
394
+	trace_btrfs_tree_read_unlock_blocking(eb);
206
395
 	/*
207
396
 	 * if we're nested, we have the write lock.  No new locking
208
397
 	 * is needed as long as we are the lock owner.
209
-	 * The write unlock will do a barrier for us, and the lock_nested
398
+	 * The write unlock will do a barrier for us, and the lock_recursed
210
399
 	 * field only matters to the lock owner.
211
400
 	 */
212
-	if (eb->lock_nested && current->pid == eb->lock_owner) {
213
-		eb->lock_nested = 0;
401
+	if (eb->lock_recursed && current->pid == eb->lock_owner) {
402
+		eb->lock_recursed = false;
214
403
 		return;
215
404
 	}
216
405
 	btrfs_assert_tree_read_locked(eb);
..
..
@@ -218,70 +407,268 @@
218
407
 	/* atomic_dec_and_test implies a barrier */
219
408
 	if (atomic_dec_and_test(&eb->blocking_readers))
220
409
 		cond_wake_up_nomb(&eb->read_lock_wq);
221
-	atomic_dec(&eb->read_locks);
410
+	btrfs_assert_tree_read_locks_put(eb);
222
411
 }
223
412
 
224
413
 /*
225
- * take a spinning write lock.  This will wait for both
226
- * blocking readers or writers
414
+ * Lock for write. Wait for all blocking and spinning readers and writers. This
415
+ * starts context where reader lock could be nested by the same thread.
416
+ *
417
+ * The rwlock is held for write upon exit.
227
418
  */
228
-void btrfs_tree_lock(struct extent_buffer *eb)
419
+void __btrfs_tree_lock(struct extent_buffer *eb, enum btrfs_lock_nesting nest)
420
+	__acquires(&eb->lock)
229
421
 {
422
+	u64 start_ns = 0;
423
+
424
+	if (trace_btrfs_tree_lock_enabled())
425
+		start_ns = ktime_get_ns();
426
+
230
427
 	WARN_ON(eb->lock_owner == current->pid);
231
428
 again:
232
429
 	wait_event(eb->read_lock_wq, atomic_read(&eb->blocking_readers) == 0);
233
-	wait_event(eb->write_lock_wq, atomic_read(&eb->blocking_writers) == 0);
430
+	wait_event(eb->write_lock_wq, READ_ONCE(eb->blocking_writers) == 0);
234
431
 	write_lock(&eb->lock);
235
-	if (atomic_read(&eb->blocking_readers)) {
432
+	/* Refetch value after lock */
433
+	if (atomic_read(&eb->blocking_readers) ||
434
+	    READ_ONCE(eb->blocking_writers)) {
236
435
 		write_unlock(&eb->lock);
237
-		wait_event(eb->read_lock_wq,
238
-			   atomic_read(&eb->blocking_readers) == 0);
239
436
 		goto again;
240
437
 	}
241
-	if (atomic_read(&eb->blocking_writers)) {
242
-		write_unlock(&eb->lock);
243
-		wait_event(eb->write_lock_wq,
244
-			   atomic_read(&eb->blocking_writers) == 0);
245
-		goto again;
246
-	}
247
-	WARN_ON(atomic_read(&eb->spinning_writers));
248
-	atomic_inc(&eb->spinning_writers);
249
-	atomic_inc(&eb->write_locks);
438
+	btrfs_assert_spinning_writers_get(eb);
439
+	btrfs_assert_tree_write_locks_get(eb);
250
440
 	eb->lock_owner = current->pid;
441
+	trace_btrfs_tree_lock(eb, start_ns);
442
+}
443
+
444
+void btrfs_tree_lock(struct extent_buffer *eb)
445
+{
446
+	__btrfs_tree_lock(eb, BTRFS_NESTING_NORMAL);
251
447
 }
252
448
 
253
449
 /*
254
- * drop a spinning or a blocking write lock.
450
+ * Release the write lock, either blocking or spinning (ie. there's no need
451
+ * for an explicit blocking unlock, like btrfs_tree_read_unlock_blocking).
452
+ * This also ends the context for nesting, the read lock must have been
453
+ * released already.
454
+ *
455
+ * Tasks blocked and waiting are woken, rwlock is not held upon exit.
255
456
  */
256
457
 void btrfs_tree_unlock(struct extent_buffer *eb)
257
458
 {
258
-	int blockers = atomic_read(&eb->blocking_writers);
459
+	/*
460
+	 * This is read both locked and unlocked but always by the same thread
461
+	 * that already owns the lock so we don't need to use READ_ONCE
462
+	 */
463
+	int blockers = eb->blocking_writers;
259
464
 
260
465
 	BUG_ON(blockers > 1);
261
466
 
262
467
 	btrfs_assert_tree_locked(eb);
468
+	trace_btrfs_tree_unlock(eb);
263
469
 	eb->lock_owner = 0;
264
-	atomic_dec(&eb->write_locks);
470
+	btrfs_assert_tree_write_locks_put(eb);
265
471
 
266
472
 	if (blockers) {
267
-		WARN_ON(atomic_read(&eb->spinning_writers));
268
-		atomic_dec(&eb->blocking_writers);
269
-		/* Use the lighter barrier after atomic */
270
-		smp_mb__after_atomic();
271
-		cond_wake_up_nomb(&eb->write_lock_wq);
473
+		btrfs_assert_no_spinning_writers(eb);
474
+		/* Unlocked write */
475
+		WRITE_ONCE(eb->blocking_writers, 0);
476
+		/*
477
+		 * We need to order modifying blocking_writers above with
478
+		 * actually waking up the sleepers to ensure they see the
479
+		 * updated value of blocking_writers
480
+		 */
481
+		cond_wake_up(&eb->write_lock_wq);
272
482
 	} else {
273
-		WARN_ON(atomic_read(&eb->spinning_writers) != 1);
274
-		atomic_dec(&eb->spinning_writers);
483
+		btrfs_assert_spinning_writers_put(eb);
275
484
 		write_unlock(&eb->lock);
276
485
 	}
277
486
 }
278
487
 
279
-void btrfs_assert_tree_locked(struct extent_buffer *eb)
488
+/*
489
+ * Set all locked nodes in the path to blocking locks.  This should be done
490
+ * before scheduling
491
+ */
492
+void btrfs_set_path_blocking(struct btrfs_path *p)
280
493
 {
281
-	BUG_ON(!atomic_read(&eb->write_locks));
494
+	int i;
495
+
496
+	for (i = 0; i < BTRFS_MAX_LEVEL; i++) {
497
+		if (!p->nodes[i] || !p->locks[i])
498
+			continue;
499
+		/*
500
+		 * If we currently have a spinning reader or writer lock this
501
+		 * will bump the count of blocking holders and drop the
502
+		 * spinlock.
503
+		 */
504
+		if (p->locks[i] == BTRFS_READ_LOCK) {
505
+			btrfs_set_lock_blocking_read(p->nodes[i]);
506
+			p->locks[i] = BTRFS_READ_LOCK_BLOCKING;
507
+		} else if (p->locks[i] == BTRFS_WRITE_LOCK) {
508
+			btrfs_set_lock_blocking_write(p->nodes[i]);
509
+			p->locks[i] = BTRFS_WRITE_LOCK_BLOCKING;
510
+		}
511
+	}
282
512
 }
283
513
 
284
-static void btrfs_assert_tree_read_locked(struct extent_buffer *eb)
514
+/*
515
+ * This releases any locks held in the path starting at level and going all the
516
+ * way up to the root.
517
+ *
518
+ * btrfs_search_slot will keep the lock held on higher nodes in a few corner
519
+ * cases, such as COW of the block at slot zero in the node.  This ignores
520
+ * those rules, and it should only be called when there are no more updates to
521
+ * be done higher up in the tree.
522
+ */
523
+void btrfs_unlock_up_safe(struct btrfs_path *path, int level)
285
524
 {
286
-	BUG_ON(!atomic_read(&eb->read_locks));
525
+	int i;
526
+
527
+	if (path->keep_locks)
528
+		return;
529
+
530
+	for (i = level; i < BTRFS_MAX_LEVEL; i++) {
531
+		if (!path->nodes[i])
532
+			continue;
533
+		if (!path->locks[i])
534
+			continue;
535
+		btrfs_tree_unlock_rw(path->nodes[i], path->locks[i]);
536
+		path->locks[i] = 0;
537
+	}
538
+}
539
+
540
+/*
541
+ * Loop around taking references on and locking the root node of the tree until
542
+ * we end up with a lock on the root node.
543
+ *
544
+ * Return: root extent buffer with write lock held
545
+ */
546
+struct extent_buffer *btrfs_lock_root_node(struct btrfs_root *root)
547
+{
548
+	struct extent_buffer *eb;
549
+
550
+	while (1) {
551
+		eb = btrfs_root_node(root);
552
+		btrfs_tree_lock(eb);
553
+		if (eb == root->node)
554
+			break;
555
+		btrfs_tree_unlock(eb);
556
+		free_extent_buffer(eb);
557
+	}
558
+	return eb;
559
+}
560
+
561
+/*
562
+ * Loop around taking references on and locking the root node of the tree until
563
+ * we end up with a lock on the root node.
564
+ *
565
+ * Return: root extent buffer with read lock held
566
+ */
567
+struct extent_buffer *__btrfs_read_lock_root_node(struct btrfs_root *root,
568
+						  bool recurse)
569
+{
570
+	struct extent_buffer *eb;
571
+
572
+	while (1) {
573
+		eb = btrfs_root_node(root);
574
+		__btrfs_tree_read_lock(eb, BTRFS_NESTING_NORMAL, recurse);
575
+		if (eb == root->node)
576
+			break;
577
+		btrfs_tree_read_unlock(eb);
578
+		free_extent_buffer(eb);
579
+	}
580
+	return eb;
581
+}
582
+
583
+/*
584
+ * DREW locks
585
+ * ==========
586
+ *
587
+ * DREW stands for double-reader-writer-exclusion lock. It's used in situation
588
+ * where you want to provide A-B exclusion but not AA or BB.
589
+ *
590
+ * Currently implementation gives more priority to reader. If a reader and a
591
+ * writer both race to acquire their respective sides of the lock the writer
592
+ * would yield its lock as soon as it detects a concurrent reader. Additionally
593
+ * if there are pending readers no new writers would be allowed to come in and
594
+ * acquire the lock.
595
+ */
596
+
597
+int btrfs_drew_lock_init(struct btrfs_drew_lock *lock)
598
+{
599
+	int ret;
600
+
601
+	ret = percpu_counter_init(&lock->writers, 0, GFP_KERNEL);
602
+	if (ret)
603
+		return ret;
604
+
605
+	atomic_set(&lock->readers, 0);
606
+	init_waitqueue_head(&lock->pending_readers);
607
+	init_waitqueue_head(&lock->pending_writers);
608
+
609
+	return 0;
610
+}
611
+
612
+void btrfs_drew_lock_destroy(struct btrfs_drew_lock *lock)
613
+{
614
+	percpu_counter_destroy(&lock->writers);
615
+}
616
+
617
+/* Return true if acquisition is successful, false otherwise */
618
+bool btrfs_drew_try_write_lock(struct btrfs_drew_lock *lock)
619
+{
620
+	if (atomic_read(&lock->readers))
621
+		return false;
622
+
623
+	percpu_counter_inc(&lock->writers);
624
+
625
+	/* Ensure writers count is updated before we check for pending readers */
626
+	smp_mb();
627
+	if (atomic_read(&lock->readers)) {
628
+		btrfs_drew_write_unlock(lock);
629
+		return false;
630
+	}
631
+
632
+	return true;
633
+}
634
+
635
+void btrfs_drew_write_lock(struct btrfs_drew_lock *lock)
636
+{
637
+	while (true) {
638
+		if (btrfs_drew_try_write_lock(lock))
639
+			return;
640
+		wait_event(lock->pending_writers, !atomic_read(&lock->readers));
641
+	}
642
+}
643
+
644
+void btrfs_drew_write_unlock(struct btrfs_drew_lock *lock)
645
+{
646
+	percpu_counter_dec(&lock->writers);
647
+	cond_wake_up(&lock->pending_readers);
648
+}
649
+
650
+void btrfs_drew_read_lock(struct btrfs_drew_lock *lock)
651
+{
652
+	atomic_inc(&lock->readers);
653
+
654
+	/*
655
+	 * Ensure the pending reader count is perceieved BEFORE this reader
656
+	 * goes to sleep in case of active writers. This guarantees new writers
657
+	 * won't be allowed and that the current reader will be woken up when
658
+	 * the last active writer finishes its jobs.
659
+	 */
660
+	smp_mb__after_atomic();
661
+
662
+	wait_event(lock->pending_readers,
663
+		   percpu_counter_sum(&lock->writers) == 0);
664
+}
665
+
666
+void btrfs_drew_read_unlock(struct btrfs_drew_lock *lock)
667
+{
668
+	/*
669
+	 * atomic_dec_and_test implies a full barrier, so woken up writers
670
+	 * are guaranteed to see the decrement
671
+	 */
672
+	if (atomic_dec_and_test(&lock->readers))
673
+		wake_up(&lock->pending_writers);
287
674
 }