| .. | .. |
|---|
| 12 | 12 | #include "async-thread.h" |
|---|
| 13 | 13 | #include "ctree.h" |
|---|
| 14 | 14 | |
|---|
| 15 | | -#define WORK_DONE_BIT 0 |
|---|
| 16 | | -#define WORK_ORDER_DONE_BIT 1 |
|---|
| 17 | | -#define WORK_HIGH_PRIO_BIT 2 |
|---|
| 15 | +enum { |
|---|
| 16 | + WORK_DONE_BIT, |
|---|
| 17 | + WORK_ORDER_DONE_BIT, |
|---|
| 18 | + WORK_HIGH_PRIO_BIT, |
|---|
| 19 | +}; |
|---|
| 18 | 20 | |
|---|
| 19 | 21 | #define NO_THRESHOLD (-1) |
|---|
| 20 | 22 | #define DFT_THRESHOLD (32) |
|---|
| .. | .. |
|---|
| 51 | 53 | struct __btrfs_workqueue *high; |
|---|
| 52 | 54 | }; |
|---|
| 53 | 55 | |
|---|
| 54 | | -static void normal_work_helper(struct btrfs_work *work); |
|---|
| 55 | | - |
|---|
| 56 | | -#define BTRFS_WORK_HELPER(name) \ |
|---|
| 57 | | -noinline_for_stack void btrfs_##name(struct work_struct *arg) \ |
|---|
| 58 | | -{ \ |
|---|
| 59 | | - struct btrfs_work *work = container_of(arg, struct btrfs_work, \ |
|---|
| 60 | | - normal_work); \ |
|---|
| 61 | | - normal_work_helper(work); \ |
|---|
| 62 | | -} |
|---|
| 63 | | - |
|---|
| 64 | | -struct btrfs_fs_info * |
|---|
| 65 | | -btrfs_workqueue_owner(const struct __btrfs_workqueue *wq) |
|---|
| 56 | +struct btrfs_fs_info * __pure btrfs_workqueue_owner(const struct __btrfs_workqueue *wq) |
|---|
| 66 | 57 | { |
|---|
| 67 | 58 | return wq->fs_info; |
|---|
| 68 | 59 | } |
|---|
| 69 | 60 | |
|---|
| 70 | | -struct btrfs_fs_info * |
|---|
| 71 | | -btrfs_work_owner(const struct btrfs_work *work) |
|---|
| 61 | +struct btrfs_fs_info * __pure btrfs_work_owner(const struct btrfs_work *work) |
|---|
| 72 | 62 | { |
|---|
| 73 | 63 | return work->wq->fs_info; |
|---|
| 74 | 64 | } |
|---|
| .. | .. |
|---|
| 86 | 76 | |
|---|
| 87 | 77 | return atomic_read(&wq->normal->pending) > wq->normal->thresh * 2; |
|---|
| 88 | 78 | } |
|---|
| 89 | | - |
|---|
| 90 | | -BTRFS_WORK_HELPER(worker_helper); |
|---|
| 91 | | -BTRFS_WORK_HELPER(delalloc_helper); |
|---|
| 92 | | -BTRFS_WORK_HELPER(flush_delalloc_helper); |
|---|
| 93 | | -BTRFS_WORK_HELPER(cache_helper); |
|---|
| 94 | | -BTRFS_WORK_HELPER(submit_helper); |
|---|
| 95 | | -BTRFS_WORK_HELPER(fixup_helper); |
|---|
| 96 | | -BTRFS_WORK_HELPER(endio_helper); |
|---|
| 97 | | -BTRFS_WORK_HELPER(endio_meta_helper); |
|---|
| 98 | | -BTRFS_WORK_HELPER(endio_meta_write_helper); |
|---|
| 99 | | -BTRFS_WORK_HELPER(endio_raid56_helper); |
|---|
| 100 | | -BTRFS_WORK_HELPER(endio_repair_helper); |
|---|
| 101 | | -BTRFS_WORK_HELPER(rmw_helper); |
|---|
| 102 | | -BTRFS_WORK_HELPER(endio_write_helper); |
|---|
| 103 | | -BTRFS_WORK_HELPER(freespace_write_helper); |
|---|
| 104 | | -BTRFS_WORK_HELPER(delayed_meta_helper); |
|---|
| 105 | | -BTRFS_WORK_HELPER(readahead_helper); |
|---|
| 106 | | -BTRFS_WORK_HELPER(qgroup_rescan_helper); |
|---|
| 107 | | -BTRFS_WORK_HELPER(extent_refs_helper); |
|---|
| 108 | | -BTRFS_WORK_HELPER(scrub_helper); |
|---|
| 109 | | -BTRFS_WORK_HELPER(scrubwrc_helper); |
|---|
| 110 | | -BTRFS_WORK_HELPER(scrubnc_helper); |
|---|
| 111 | | -BTRFS_WORK_HELPER(scrubparity_helper); |
|---|
| 112 | 79 | |
|---|
| 113 | 80 | static struct __btrfs_workqueue * |
|---|
| 114 | 81 | __btrfs_alloc_workqueue(struct btrfs_fs_info *fs_info, const char *name, |
|---|
| .. | .. |
|---|
| 139 | 106 | } |
|---|
| 140 | 107 | |
|---|
| 141 | 108 | if (flags & WQ_HIGHPRI) |
|---|
| 142 | | - ret->normal_wq = alloc_workqueue("%s-%s-high", flags, |
|---|
| 143 | | - ret->current_active, "btrfs", |
|---|
| 144 | | - name); |
|---|
| 109 | + ret->normal_wq = alloc_workqueue("btrfs-%s-high", flags, |
|---|
| 110 | + ret->current_active, name); |
|---|
| 145 | 111 | else |
|---|
| 146 | | - ret->normal_wq = alloc_workqueue("%s-%s", flags, |
|---|
| 147 | | - ret->current_active, "btrfs", |
|---|
| 148 | | - name); |
|---|
| 112 | + ret->normal_wq = alloc_workqueue("btrfs-%s", flags, |
|---|
| 113 | + ret->current_active, name); |
|---|
| 149 | 114 | if (!ret->normal_wq) { |
|---|
| 150 | 115 | kfree(ret); |
|---|
| 151 | 116 | return NULL; |
|---|
| .. | .. |
|---|
| 259 | 224 | struct btrfs_work *work; |
|---|
| 260 | 225 | spinlock_t *lock = &wq->list_lock; |
|---|
| 261 | 226 | unsigned long flags; |
|---|
| 262 | | - void *wtag; |
|---|
| 263 | 227 | bool free_self = false; |
|---|
| 264 | 228 | |
|---|
| 265 | 229 | while (1) { |
|---|
| .. | .. |
|---|
| 309 | 273 | * original work item cannot depend on the recycled work |
|---|
| 310 | 274 | * item in that case (see find_worker_executing_work()). |
|---|
| 311 | 275 | * |
|---|
| 312 | | - * Note that the work of one Btrfs filesystem may depend |
|---|
| 313 | | - * on the work of another Btrfs filesystem via, e.g., a |
|---|
| 314 | | - * loop device. Therefore, we must not allow the current |
|---|
| 315 | | - * work item to be recycled until we are really done, |
|---|
| 316 | | - * otherwise we break the above assumption and can |
|---|
| 317 | | - * deadlock. |
|---|
| 276 | + * Note that different types of Btrfs work can depend on |
|---|
| 277 | + * each other, and one type of work on one Btrfs |
|---|
| 278 | + * filesystem may even depend on the same type of work |
|---|
| 279 | + * on another Btrfs filesystem via, e.g., a loop device. |
|---|
| 280 | + * Therefore, we must not allow the current work item to |
|---|
| 281 | + * be recycled until we are really done, otherwise we |
|---|
| 282 | + * break the above assumption and can deadlock. |
|---|
| 318 | 283 | */ |
|---|
| 319 | 284 | free_self = true; |
|---|
| 320 | 285 | } else { |
|---|
| 321 | 286 | /* |
|---|
| 322 | 287 | * We don't want to call the ordered free functions with |
|---|
| 323 | | - * the lock held though. Save the work as tag for the |
|---|
| 324 | | - * trace event, because the callback could free the |
|---|
| 325 | | - * structure. |
|---|
| 288 | + * the lock held. |
|---|
| 326 | 289 | */ |
|---|
| 327 | | - wtag = work; |
|---|
| 328 | 290 | work->ordered_free(work); |
|---|
| 329 | | - trace_btrfs_all_work_done(wq->fs_info, wtag); |
|---|
| 291 | + /* NB: work must not be dereferenced past this point. */ |
|---|
| 292 | + trace_btrfs_all_work_done(wq->fs_info, work); |
|---|
| 330 | 293 | } |
|---|
| 331 | 294 | } |
|---|
| 332 | 295 | spin_unlock_irqrestore(lock, flags); |
|---|
| 333 | 296 | |
|---|
| 334 | 297 | if (free_self) { |
|---|
| 335 | | - wtag = self; |
|---|
| 336 | 298 | self->ordered_free(self); |
|---|
| 337 | | - trace_btrfs_all_work_done(wq->fs_info, wtag); |
|---|
| 299 | + /* NB: self must not be dereferenced past this point. */ |
|---|
| 300 | + trace_btrfs_all_work_done(wq->fs_info, self); |
|---|
| 338 | 301 | } |
|---|
| 339 | 302 | } |
|---|
| 340 | 303 | |
|---|
| 341 | | -static void normal_work_helper(struct btrfs_work *work) |
|---|
| 304 | +static void btrfs_work_helper(struct work_struct *normal_work) |
|---|
| 342 | 305 | { |
|---|
| 306 | + struct btrfs_work *work = container_of(normal_work, struct btrfs_work, |
|---|
| 307 | + normal_work); |
|---|
| 343 | 308 | struct __btrfs_workqueue *wq; |
|---|
| 344 | | - void *wtag; |
|---|
| 345 | 309 | int need_order = 0; |
|---|
| 346 | 310 | |
|---|
| 347 | 311 | /* |
|---|
| .. | .. |
|---|
| 355 | 319 | if (work->ordered_func) |
|---|
| 356 | 320 | need_order = 1; |
|---|
| 357 | 321 | wq = work->wq; |
|---|
| 358 | | - /* Safe for tracepoints in case work gets freed by the callback */ |
|---|
| 359 | | - wtag = work; |
|---|
| 360 | 322 | |
|---|
| 361 | 323 | trace_btrfs_work_sched(work); |
|---|
| 362 | 324 | thresh_exec_hook(wq); |
|---|
| .. | .. |
|---|
| 371 | 333 | smp_mb__before_atomic(); |
|---|
| 372 | 334 | set_bit(WORK_DONE_BIT, &work->flags); |
|---|
| 373 | 335 | run_ordered_work(wq, work); |
|---|
| 336 | + } else { |
|---|
| 337 | + /* NB: work must not be dereferenced past this point. */ |
|---|
| 338 | + trace_btrfs_all_work_done(wq->fs_info, work); |
|---|
| 374 | 339 | } |
|---|
| 375 | | - if (!need_order) |
|---|
| 376 | | - trace_btrfs_all_work_done(wq->fs_info, wtag); |
|---|
| 377 | 340 | } |
|---|
| 378 | 341 | |
|---|
| 379 | | -void btrfs_init_work(struct btrfs_work *work, btrfs_work_func_t uniq_func, |
|---|
| 380 | | - btrfs_func_t func, |
|---|
| 381 | | - btrfs_func_t ordered_func, |
|---|
| 382 | | - btrfs_func_t ordered_free) |
|---|
| 342 | +void btrfs_init_work(struct btrfs_work *work, btrfs_func_t func, |
|---|
| 343 | + btrfs_func_t ordered_func, btrfs_func_t ordered_free) |
|---|
| 383 | 344 | { |
|---|
| 384 | 345 | work->func = func; |
|---|
| 385 | 346 | work->ordered_func = ordered_func; |
|---|
| 386 | 347 | work->ordered_free = ordered_free; |
|---|
| 387 | | - INIT_WORK(&work->normal_work, uniq_func); |
|---|
| 348 | + INIT_WORK(&work->normal_work, btrfs_work_helper); |
|---|
| 388 | 349 | INIT_LIST_HEAD(&work->ordered_list); |
|---|
| 389 | 350 | work->flags = 0; |
|---|
| 390 | 351 | } |
|---|