hc
2024-10-22 8ac6c7a54ed1b98d142dce24b11c6de6a1e239a5
kernel/block/blk-ioc.c
....@@ -9,7 +9,6 @@
99 #include <linux/blkdev.h>
1010 #include <linux/slab.h>
1111 #include <linux/sched/task.h>
12
-#include <linux/delay.h>
1312
1413 #include "blk.h"
1514
....@@ -29,7 +28,6 @@
2928 BUG_ON(atomic_long_read(&ioc->refcount) <= 0);
3029 atomic_long_inc(&ioc->refcount);
3130 }
32
-EXPORT_SYMBOL(get_io_context);
3331
3432 static void icq_free_icq_rcu(struct rcu_head *head)
3533 {
....@@ -49,10 +47,8 @@
4947 if (icq->flags & ICQ_EXITED)
5048 return;
5149
52
- if (et->uses_mq && et->ops.mq.exit_icq)
53
- et->ops.mq.exit_icq(icq);
54
- else if (!et->uses_mq && et->ops.sq.elevator_exit_icq_fn)
55
- et->ops.sq.elevator_exit_icq_fn(icq);
50
+ if (et->ops.exit_icq)
51
+ et->ops.exit_icq(icq);
5652
5753 icq->flags |= ICQ_EXITED;
5854 }
....@@ -100,32 +96,38 @@
10096 {
10197 struct io_context *ioc = container_of(work, struct io_context,
10298 release_work);
103
- unsigned long flags;
104
-
105
- /*
106
- * Exiting icq may call into put_io_context() through elevator
107
- * which will trigger lockdep warning. The ioc's are guaranteed to
108
- * be different, use a different locking subclass here. Use
109
- * irqsave variant as there's no spin_lock_irq_nested().
110
- */
111
- spin_lock_irqsave_nested(&ioc->lock, flags, 1);
99
+ spin_lock_irq(&ioc->lock);
112100
113101 while (!hlist_empty(&ioc->icq_list)) {
114102 struct io_cq *icq = hlist_entry(ioc->icq_list.first,
115103 struct io_cq, ioc_node);
116104 struct request_queue *q = icq->q;
117105
118
- if (spin_trylock(q->queue_lock)) {
106
+ if (spin_trylock(&q->queue_lock)) {
119107 ioc_destroy_icq(icq);
120
- spin_unlock(q->queue_lock);
108
+ spin_unlock(&q->queue_lock);
121109 } else {
122
- spin_unlock_irqrestore(&ioc->lock, flags);
123
- cpu_chill();
124
- spin_lock_irqsave_nested(&ioc->lock, flags, 1);
110
+ /* Make sure q and icq cannot be freed. */
111
+ rcu_read_lock();
112
+
113
+ /* Re-acquire the locks in the correct order. */
114
+ spin_unlock(&ioc->lock);
115
+ spin_lock(&q->queue_lock);
116
+ spin_lock(&ioc->lock);
117
+
118
+ /*
119
+ * The icq may have been destroyed when the ioc lock
120
+ * was released.
121
+ */
122
+ if (!(icq->flags & ICQ_DESTROYED))
123
+ ioc_destroy_icq(icq);
124
+
125
+ spin_unlock(&q->queue_lock);
126
+ rcu_read_unlock();
125127 }
126128 }
127129
128
- spin_unlock_irqrestore(&ioc->lock, flags);
130
+ spin_unlock_irq(&ioc->lock);
129131
130132 kmem_cache_free(iocontext_cachep, ioc);
131133 }
....@@ -164,7 +166,6 @@
164166 if (free_ioc)
165167 kmem_cache_free(iocontext_cachep, ioc);
166168 }
167
-EXPORT_SYMBOL(put_io_context);
168169
169170 /**
170171 * put_io_context_active - put active reference on ioc
....@@ -175,8 +176,6 @@
175176 */
176177 void put_io_context_active(struct io_context *ioc)
177178 {
178
- struct elevator_type *et;
179
- unsigned long flags;
180179 struct io_cq *icq;
181180
182181 if (!atomic_dec_and_test(&ioc->active_ref)) {
....@@ -184,32 +183,14 @@
184183 return;
185184 }
186185
187
- /*
188
- * Need ioc lock to walk icq_list and q lock to exit icq. Perform
189
- * reverse double locking. Read comment in ioc_release_fn() for
190
- * explanation on the nested locking annotation.
191
- */
192
-retry:
193
- spin_lock_irqsave_nested(&ioc->lock, flags, 1);
186
+ spin_lock_irq(&ioc->lock);
194187 hlist_for_each_entry(icq, &ioc->icq_list, ioc_node) {
195188 if (icq->flags & ICQ_EXITED)
196189 continue;
197190
198
- et = icq->q->elevator->type;
199
- if (et->uses_mq) {
200
- ioc_exit_icq(icq);
201
- } else {
202
- if (spin_trylock(icq->q->queue_lock)) {
203
- ioc_exit_icq(icq);
204
- spin_unlock(icq->q->queue_lock);
205
- } else {
206
- spin_unlock_irqrestore(&ioc->lock, flags);
207
- cpu_chill();
208
- goto retry;
209
- }
210
- }
191
+ ioc_exit_icq(icq);
211192 }
212
- spin_unlock_irqrestore(&ioc->lock, flags);
193
+ spin_unlock_irq(&ioc->lock);
213194
214195 put_io_context(ioc);
215196 }
....@@ -235,7 +216,7 @@
235216 rcu_read_lock();
236217 while (!list_empty(icq_list)) {
237218 struct io_cq *icq = list_entry(icq_list->next,
238
- struct io_cq, q_node);
219
+ struct io_cq, q_node);
239220 struct io_context *ioc = icq->ioc;
240221
241222 spin_lock_irqsave(&ioc->lock, flags);
....@@ -259,16 +240,11 @@
259240 {
260241 LIST_HEAD(icq_list);
261242
262
- spin_lock_irq(q->queue_lock);
243
+ spin_lock_irq(&q->queue_lock);
263244 list_splice_init(&q->icq_list, &icq_list);
245
+ spin_unlock_irq(&q->queue_lock);
264246
265
- if (q->mq_ops) {
266
- spin_unlock_irq(q->queue_lock);
267
- __ioc_clear_queue(&icq_list);
268
- } else {
269
- __ioc_clear_queue(&icq_list);
270
- spin_unlock_irq(q->queue_lock);
271
- }
247
+ __ioc_clear_queue(&icq_list);
272248 }
273249
274250 int create_task_io_context(struct task_struct *task, gfp_t gfp_flags, int node)
....@@ -344,7 +320,6 @@
344320
345321 return NULL;
346322 }
347
-EXPORT_SYMBOL(get_task_io_context);
348323
349324 /**
350325 * ioc_lookup_icq - lookup io_cq from ioc
....@@ -358,7 +333,7 @@
358333 {
359334 struct io_cq *icq;
360335
361
- lockdep_assert_held(q->queue_lock);
336
+ lockdep_assert_held(&q->queue_lock);
362337
363338 /*
364339 * icq's are indexed from @ioc using radix tree and hint pointer,
....@@ -417,16 +392,14 @@
417392 INIT_HLIST_NODE(&icq->ioc_node);
418393
419394 /* lock both q and ioc and try to link @icq */
420
- spin_lock_irq(q->queue_lock);
395
+ spin_lock_irq(&q->queue_lock);
421396 spin_lock(&ioc->lock);
422397
423398 if (likely(!radix_tree_insert(&ioc->icq_tree, q->id, icq))) {
424399 hlist_add_head(&icq->ioc_node, &ioc->icq_list);
425400 list_add(&icq->q_node, &q->icq_list);
426
- if (et->uses_mq && et->ops.mq.init_icq)
427
- et->ops.mq.init_icq(icq);
428
- else if (!et->uses_mq && et->ops.sq.elevator_init_icq_fn)
429
- et->ops.sq.elevator_init_icq_fn(icq);
401
+ if (et->ops.init_icq)
402
+ et->ops.init_icq(icq);
430403 } else {
431404 kmem_cache_free(et->icq_cache, icq);
432405 icq = ioc_lookup_icq(ioc, q);
....@@ -435,7 +408,7 @@
435408 }
436409
437410 spin_unlock(&ioc->lock);
438
- spin_unlock_irq(q->queue_lock);
411
+ spin_unlock_irq(&q->queue_lock);
439412 radix_tree_preload_end();
440413 return icq;
441414 }