.. | .. |
---|
12 | 12 | #include "async-thread.h" |
---|
13 | 13 | #include "ctree.h" |
---|
14 | 14 | |
---|
15 | | -#define WORK_DONE_BIT 0 |
---|
16 | | -#define WORK_ORDER_DONE_BIT 1 |
---|
17 | | -#define WORK_HIGH_PRIO_BIT 2 |
---|
| 15 | +enum { |
---|
| 16 | + WORK_DONE_BIT, |
---|
| 17 | + WORK_ORDER_DONE_BIT, |
---|
| 18 | + WORK_HIGH_PRIO_BIT, |
---|
| 19 | +}; |
---|
18 | 20 | |
---|
19 | 21 | #define NO_THRESHOLD (-1) |
---|
20 | 22 | #define DFT_THRESHOLD (32) |
---|
.. | .. |
---|
51 | 53 | struct __btrfs_workqueue *high; |
---|
52 | 54 | }; |
---|
53 | 55 | |
---|
54 | | -static void normal_work_helper(struct btrfs_work *work); |
---|
55 | | - |
---|
56 | | -#define BTRFS_WORK_HELPER(name) \ |
---|
57 | | -noinline_for_stack void btrfs_##name(struct work_struct *arg) \ |
---|
58 | | -{ \ |
---|
59 | | - struct btrfs_work *work = container_of(arg, struct btrfs_work, \ |
---|
60 | | - normal_work); \ |
---|
61 | | - normal_work_helper(work); \ |
---|
62 | | -} |
---|
63 | | - |
---|
64 | | -struct btrfs_fs_info * |
---|
65 | | -btrfs_workqueue_owner(const struct __btrfs_workqueue *wq) |
---|
| 56 | +struct btrfs_fs_info * __pure btrfs_workqueue_owner(const struct __btrfs_workqueue *wq) |
---|
66 | 57 | { |
---|
67 | 58 | return wq->fs_info; |
---|
68 | 59 | } |
---|
69 | 60 | |
---|
70 | | -struct btrfs_fs_info * |
---|
71 | | -btrfs_work_owner(const struct btrfs_work *work) |
---|
| 61 | +struct btrfs_fs_info * __pure btrfs_work_owner(const struct btrfs_work *work) |
---|
72 | 62 | { |
---|
73 | 63 | return work->wq->fs_info; |
---|
74 | 64 | } |
---|
.. | .. |
---|
86 | 76 | |
---|
87 | 77 | return atomic_read(&wq->normal->pending) > wq->normal->thresh * 2; |
---|
88 | 78 | } |
---|
89 | | - |
---|
90 | | -BTRFS_WORK_HELPER(worker_helper); |
---|
91 | | -BTRFS_WORK_HELPER(delalloc_helper); |
---|
92 | | -BTRFS_WORK_HELPER(flush_delalloc_helper); |
---|
93 | | -BTRFS_WORK_HELPER(cache_helper); |
---|
94 | | -BTRFS_WORK_HELPER(submit_helper); |
---|
95 | | -BTRFS_WORK_HELPER(fixup_helper); |
---|
96 | | -BTRFS_WORK_HELPER(endio_helper); |
---|
97 | | -BTRFS_WORK_HELPER(endio_meta_helper); |
---|
98 | | -BTRFS_WORK_HELPER(endio_meta_write_helper); |
---|
99 | | -BTRFS_WORK_HELPER(endio_raid56_helper); |
---|
100 | | -BTRFS_WORK_HELPER(endio_repair_helper); |
---|
101 | | -BTRFS_WORK_HELPER(rmw_helper); |
---|
102 | | -BTRFS_WORK_HELPER(endio_write_helper); |
---|
103 | | -BTRFS_WORK_HELPER(freespace_write_helper); |
---|
104 | | -BTRFS_WORK_HELPER(delayed_meta_helper); |
---|
105 | | -BTRFS_WORK_HELPER(readahead_helper); |
---|
106 | | -BTRFS_WORK_HELPER(qgroup_rescan_helper); |
---|
107 | | -BTRFS_WORK_HELPER(extent_refs_helper); |
---|
108 | | -BTRFS_WORK_HELPER(scrub_helper); |
---|
109 | | -BTRFS_WORK_HELPER(scrubwrc_helper); |
---|
110 | | -BTRFS_WORK_HELPER(scrubnc_helper); |
---|
111 | | -BTRFS_WORK_HELPER(scrubparity_helper); |
---|
112 | 79 | |
---|
113 | 80 | static struct __btrfs_workqueue * |
---|
114 | 81 | __btrfs_alloc_workqueue(struct btrfs_fs_info *fs_info, const char *name, |
---|
.. | .. |
---|
139 | 106 | } |
---|
140 | 107 | |
---|
141 | 108 | if (flags & WQ_HIGHPRI) |
---|
142 | | - ret->normal_wq = alloc_workqueue("%s-%s-high", flags, |
---|
143 | | - ret->current_active, "btrfs", |
---|
144 | | - name); |
---|
| 109 | + ret->normal_wq = alloc_workqueue("btrfs-%s-high", flags, |
---|
| 110 | + ret->current_active, name); |
---|
145 | 111 | else |
---|
146 | | - ret->normal_wq = alloc_workqueue("%s-%s", flags, |
---|
147 | | - ret->current_active, "btrfs", |
---|
148 | | - name); |
---|
| 112 | + ret->normal_wq = alloc_workqueue("btrfs-%s", flags, |
---|
| 113 | + ret->current_active, name); |
---|
149 | 114 | if (!ret->normal_wq) { |
---|
150 | 115 | kfree(ret); |
---|
151 | 116 | return NULL; |
---|
.. | .. |
---|
259 | 224 | struct btrfs_work *work; |
---|
260 | 225 | spinlock_t *lock = &wq->list_lock; |
---|
261 | 226 | unsigned long flags; |
---|
262 | | - void *wtag; |
---|
263 | 227 | bool free_self = false; |
---|
264 | 228 | |
---|
265 | 229 | while (1) { |
---|
.. | .. |
---|
309 | 273 | * original work item cannot depend on the recycled work |
---|
310 | 274 | * item in that case (see find_worker_executing_work()). |
---|
311 | 275 | * |
---|
312 | | - * Note that the work of one Btrfs filesystem may depend |
---|
313 | | - * on the work of another Btrfs filesystem via, e.g., a |
---|
314 | | - * loop device. Therefore, we must not allow the current |
---|
315 | | - * work item to be recycled until we are really done, |
---|
316 | | - * otherwise we break the above assumption and can |
---|
317 | | - * deadlock. |
---|
| 276 | + * Note that different types of Btrfs work can depend on |
---|
| 277 | + * each other, and one type of work on one Btrfs |
---|
| 278 | + * filesystem may even depend on the same type of work |
---|
| 279 | + * on another Btrfs filesystem via, e.g., a loop device. |
---|
| 280 | + * Therefore, we must not allow the current work item to |
---|
| 281 | + * be recycled until we are really done, otherwise we |
---|
| 282 | + * break the above assumption and can deadlock. |
---|
318 | 283 | */ |
---|
319 | 284 | free_self = true; |
---|
320 | 285 | } else { |
---|
321 | 286 | /* |
---|
322 | 287 | * We don't want to call the ordered free functions with |
---|
323 | | - * the lock held though. Save the work as tag for the |
---|
324 | | - * trace event, because the callback could free the |
---|
325 | | - * structure. |
---|
| 288 | + * the lock held. |
---|
326 | 289 | */ |
---|
327 | | - wtag = work; |
---|
328 | 290 | work->ordered_free(work); |
---|
329 | | - trace_btrfs_all_work_done(wq->fs_info, wtag); |
---|
| 291 | + /* NB: work must not be dereferenced past this point. */ |
---|
| 292 | + trace_btrfs_all_work_done(wq->fs_info, work); |
---|
330 | 293 | } |
---|
331 | 294 | } |
---|
332 | 295 | spin_unlock_irqrestore(lock, flags); |
---|
333 | 296 | |
---|
334 | 297 | if (free_self) { |
---|
335 | | - wtag = self; |
---|
336 | 298 | self->ordered_free(self); |
---|
337 | | - trace_btrfs_all_work_done(wq->fs_info, wtag); |
---|
| 299 | + /* NB: self must not be dereferenced past this point. */ |
---|
| 300 | + trace_btrfs_all_work_done(wq->fs_info, self); |
---|
338 | 301 | } |
---|
339 | 302 | } |
---|
340 | 303 | |
---|
341 | | -static void normal_work_helper(struct btrfs_work *work) |
---|
| 304 | +static void btrfs_work_helper(struct work_struct *normal_work) |
---|
342 | 305 | { |
---|
| 306 | + struct btrfs_work *work = container_of(normal_work, struct btrfs_work, |
---|
| 307 | + normal_work); |
---|
343 | 308 | struct __btrfs_workqueue *wq; |
---|
344 | | - void *wtag; |
---|
345 | 309 | int need_order = 0; |
---|
346 | 310 | |
---|
347 | 311 | /* |
---|
.. | .. |
---|
355 | 319 | if (work->ordered_func) |
---|
356 | 320 | need_order = 1; |
---|
357 | 321 | wq = work->wq; |
---|
358 | | - /* Safe for tracepoints in case work gets freed by the callback */ |
---|
359 | | - wtag = work; |
---|
360 | 322 | |
---|
361 | 323 | trace_btrfs_work_sched(work); |
---|
362 | 324 | thresh_exec_hook(wq); |
---|
.. | .. |
---|
371 | 333 | smp_mb__before_atomic(); |
---|
372 | 334 | set_bit(WORK_DONE_BIT, &work->flags); |
---|
373 | 335 | run_ordered_work(wq, work); |
---|
| 336 | + } else { |
---|
| 337 | + /* NB: work must not be dereferenced past this point. */ |
---|
| 338 | + trace_btrfs_all_work_done(wq->fs_info, work); |
---|
374 | 339 | } |
---|
375 | | - if (!need_order) |
---|
376 | | - trace_btrfs_all_work_done(wq->fs_info, wtag); |
---|
377 | 340 | } |
---|
378 | 341 | |
---|
379 | | -void btrfs_init_work(struct btrfs_work *work, btrfs_work_func_t uniq_func, |
---|
380 | | - btrfs_func_t func, |
---|
381 | | - btrfs_func_t ordered_func, |
---|
382 | | - btrfs_func_t ordered_free) |
---|
| 342 | +void btrfs_init_work(struct btrfs_work *work, btrfs_func_t func, |
---|
| 343 | + btrfs_func_t ordered_func, btrfs_func_t ordered_free) |
---|
383 | 344 | { |
---|
384 | 345 | work->func = func; |
---|
385 | 346 | work->ordered_func = ordered_func; |
---|
386 | 347 | work->ordered_free = ordered_free; |
---|
387 | | - INIT_WORK(&work->normal_work, uniq_func); |
---|
| 348 | + INIT_WORK(&work->normal_work, btrfs_work_helper); |
---|
388 | 349 | INIT_LIST_HEAD(&work->ordered_list); |
---|
389 | 350 | work->flags = 0; |
---|
390 | 351 | } |
---|