| .. | .. |
|---|
| 176 | 176 | * - updates to non leaf nodes just happen synchronously (see btree_split()). |
|---|
| 177 | 177 | */ |
|---|
| 178 | 178 | |
|---|
| 179 | | -#define pr_fmt(fmt) "bcache: %s() " fmt "\n", __func__ |
|---|
| 179 | +#define pr_fmt(fmt) "bcache: %s() " fmt, __func__ |
|---|
| 180 | 180 | |
|---|
| 181 | 181 | #include <linux/bcache.h> |
|---|
| 182 | 182 | #include <linux/bio.h> |
|---|
| .. | .. |
|---|
| 301 | 301 | struct block_device *bdev; |
|---|
| 302 | 302 | |
|---|
| 303 | 303 | struct cache_sb sb; |
|---|
| 304 | + struct cache_sb_disk *sb_disk; |
|---|
| 304 | 305 | struct bio sb_bio; |
|---|
| 305 | 306 | struct bio_vec sb_bv[1]; |
|---|
| 306 | 307 | struct closure sb_write; |
|---|
| .. | .. |
|---|
| 406 | 407 | struct cache { |
|---|
| 407 | 408 | struct cache_set *set; |
|---|
| 408 | 409 | struct cache_sb sb; |
|---|
| 410 | + struct cache_sb_disk *sb_disk; |
|---|
| 409 | 411 | struct bio sb_bio; |
|---|
| 410 | 412 | struct bio_vec sb_bv[1]; |
|---|
| 411 | 413 | |
|---|
| .. | .. |
|---|
| 515 | 517 | atomic_t idle_counter; |
|---|
| 516 | 518 | atomic_t at_max_writeback_rate; |
|---|
| 517 | 519 | |
|---|
| 518 | | - struct cache_sb sb; |
|---|
| 519 | | - |
|---|
| 520 | | - struct cache *cache[MAX_CACHES_PER_SET]; |
|---|
| 521 | | - struct cache *cache_by_alloc[MAX_CACHES_PER_SET]; |
|---|
| 522 | | - int caches_loaded; |
|---|
| 520 | + struct cache *cache; |
|---|
| 523 | 521 | |
|---|
| 524 | 522 | struct bcache_device **devices; |
|---|
| 525 | 523 | unsigned int devices_max_used; |
|---|
| .. | .. |
|---|
| 631 | 629 | struct bkey gc_done; |
|---|
| 632 | 630 | |
|---|
| 633 | 631 | /* |
|---|
| 632 | + * For automatical garbage collection after writeback completed, this |
|---|
| 633 | + * varialbe is used as bit fields, |
|---|
| 634 | + * - 0000 0001b (BCH_ENABLE_AUTO_GC): enable gc after writeback |
|---|
| 635 | + * - 0000 0010b (BCH_DO_AUTO_GC): do gc after writeback |
|---|
| 636 | + * This is an optimization for following write request after writeback |
|---|
| 637 | + * finished, but read hit rate dropped due to clean data on cache is |
|---|
| 638 | + * discarded. Unless user explicitly sets it via sysfs, it won't be |
|---|
| 639 | + * enabled. |
|---|
| 640 | + */ |
|---|
| 641 | +#define BCH_ENABLE_AUTO_GC 1 |
|---|
| 642 | +#define BCH_DO_AUTO_GC 2 |
|---|
| 643 | + uint8_t gc_after_writeback; |
|---|
| 644 | + |
|---|
| 645 | + /* |
|---|
| 634 | 646 | * The allocation code needs gc_mark in struct bucket to be correct, but |
|---|
| 635 | 647 | * it's not while a gc is in progress. Protected by bucket_lock. |
|---|
| 636 | 648 | */ |
|---|
| .. | .. |
|---|
| 654 | 666 | struct mutex verify_lock; |
|---|
| 655 | 667 | #endif |
|---|
| 656 | 668 | |
|---|
| 669 | + uint8_t set_uuid[16]; |
|---|
| 657 | 670 | unsigned int nr_uuids; |
|---|
| 658 | 671 | struct uuid_entry *uuids; |
|---|
| 659 | 672 | BKEY_PADDED(uuid_bucket); |
|---|
| .. | .. |
|---|
| 662 | 675 | |
|---|
| 663 | 676 | /* |
|---|
| 664 | 677 | * A btree node on disk could have too many bsets for an iterator to fit |
|---|
| 665 | | - * on the stack - have to dynamically allocate them |
|---|
| 678 | + * on the stack - have to dynamically allocate them. |
|---|
| 679 | + * bch_cache_set_alloc() will make sure the pool can allocate iterators |
|---|
| 680 | + * equipped with enough room that can host |
|---|
| 681 | + * (sb.bucket_size / sb.block_size) |
|---|
| 682 | + * btree_iter_sets, which is more than static MAX_BSETS. |
|---|
| 666 | 683 | */ |
|---|
| 667 | 684 | mempool_t fill_iter; |
|---|
| 668 | 685 | |
|---|
| .. | .. |
|---|
| 691 | 708 | atomic_long_t writeback_keys_failed; |
|---|
| 692 | 709 | |
|---|
| 693 | 710 | atomic_long_t reclaim; |
|---|
| 711 | + atomic_long_t reclaimed_journal_buckets; |
|---|
| 694 | 712 | atomic_long_t flush_write; |
|---|
| 695 | | - atomic_long_t retry_flush_write; |
|---|
| 696 | 713 | |
|---|
| 697 | 714 | enum { |
|---|
| 698 | 715 | ON_ERROR_UNREGISTER, |
|---|
| .. | .. |
|---|
| 709 | 726 | unsigned int gc_always_rewrite:1; |
|---|
| 710 | 727 | unsigned int shrinker_disabled:1; |
|---|
| 711 | 728 | unsigned int copy_gc_enabled:1; |
|---|
| 729 | + unsigned int idle_max_writeback_rate_enabled:1; |
|---|
| 712 | 730 | |
|---|
| 713 | 731 | #define BUCKET_HASH_BITS 12 |
|---|
| 714 | 732 | struct hlist_head bucket_hash[1 << BUCKET_HASH_BITS]; |
|---|
| .. | .. |
|---|
| 737 | 755 | #define btree_default_blocks(c) \ |
|---|
| 738 | 756 | ((unsigned int) ((PAGE_SECTORS * (c)->btree_pages) >> (c)->block_bits)) |
|---|
| 739 | 757 | |
|---|
| 740 | | -#define bucket_pages(c) ((c)->sb.bucket_size / PAGE_SECTORS) |
|---|
| 741 | | -#define bucket_bytes(c) ((c)->sb.bucket_size << 9) |
|---|
| 742 | | -#define block_bytes(c) ((c)->sb.block_size << 9) |
|---|
| 758 | +#define bucket_bytes(ca) ((ca)->sb.bucket_size << 9) |
|---|
| 759 | +#define block_bytes(ca) ((ca)->sb.block_size << 9) |
|---|
| 743 | 760 | |
|---|
| 744 | | -#define prios_per_bucket(c) \ |
|---|
| 745 | | - ((bucket_bytes(c) - sizeof(struct prio_set)) / \ |
|---|
| 761 | +static inline unsigned int meta_bucket_pages(struct cache_sb *sb) |
|---|
| 762 | +{ |
|---|
| 763 | + unsigned int n, max_pages; |
|---|
| 764 | + |
|---|
| 765 | + max_pages = min_t(unsigned int, |
|---|
| 766 | + __rounddown_pow_of_two(USHRT_MAX) / PAGE_SECTORS, |
|---|
| 767 | + MAX_ORDER_NR_PAGES); |
|---|
| 768 | + |
|---|
| 769 | + n = sb->bucket_size / PAGE_SECTORS; |
|---|
| 770 | + if (n > max_pages) |
|---|
| 771 | + n = max_pages; |
|---|
| 772 | + |
|---|
| 773 | + return n; |
|---|
| 774 | +} |
|---|
| 775 | + |
|---|
| 776 | +static inline unsigned int meta_bucket_bytes(struct cache_sb *sb) |
|---|
| 777 | +{ |
|---|
| 778 | + return meta_bucket_pages(sb) << PAGE_SHIFT; |
|---|
| 779 | +} |
|---|
| 780 | + |
|---|
| 781 | +#define prios_per_bucket(ca) \ |
|---|
| 782 | + ((meta_bucket_bytes(&(ca)->sb) - sizeof(struct prio_set)) / \ |
|---|
| 746 | 783 | sizeof(struct bucket_disk)) |
|---|
| 747 | | -#define prio_buckets(c) \ |
|---|
| 748 | | - DIV_ROUND_UP((size_t) (c)->sb.nbuckets, prios_per_bucket(c)) |
|---|
| 784 | + |
|---|
| 785 | +#define prio_buckets(ca) \ |
|---|
| 786 | + DIV_ROUND_UP((size_t) (ca)->sb.nbuckets, prios_per_bucket(ca)) |
|---|
| 749 | 787 | |
|---|
| 750 | 788 | static inline size_t sector_to_bucket(struct cache_set *c, sector_t s) |
|---|
| 751 | 789 | { |
|---|
| .. | .. |
|---|
| 759 | 797 | |
|---|
| 760 | 798 | static inline sector_t bucket_remainder(struct cache_set *c, sector_t s) |
|---|
| 761 | 799 | { |
|---|
| 762 | | - return s & (c->sb.bucket_size - 1); |
|---|
| 800 | + return s & (c->cache->sb.bucket_size - 1); |
|---|
| 763 | 801 | } |
|---|
| 764 | 802 | |
|---|
| 765 | 803 | static inline struct cache *PTR_CACHE(struct cache_set *c, |
|---|
| 766 | 804 | const struct bkey *k, |
|---|
| 767 | 805 | unsigned int ptr) |
|---|
| 768 | 806 | { |
|---|
| 769 | | - return c->cache[PTR_DEV(k, ptr)]; |
|---|
| 807 | + return c->cache; |
|---|
| 770 | 808 | } |
|---|
| 771 | 809 | |
|---|
| 772 | 810 | static inline size_t PTR_BUCKET_NR(struct cache_set *c, |
|---|
| .. | .. |
|---|
| 847 | 885 | |
|---|
| 848 | 886 | /* Looping macros */ |
|---|
| 849 | 887 | |
|---|
| 850 | | -#define for_each_cache(ca, cs, iter) \ |
|---|
| 851 | | - for (iter = 0; ca = cs->cache[iter], iter < (cs)->sb.nr_in_set; iter++) |
|---|
| 852 | | - |
|---|
| 853 | 888 | #define for_each_bucket(b, ca) \ |
|---|
| 854 | 889 | for (b = (ca)->buckets + (ca)->sb.first_bucket; \ |
|---|
| 855 | 890 | b < (ca)->buckets + (ca)->sb.nbuckets; b++) |
|---|
| .. | .. |
|---|
| 891 | 926 | |
|---|
| 892 | 927 | static inline void wake_up_allocators(struct cache_set *c) |
|---|
| 893 | 928 | { |
|---|
| 894 | | - struct cache *ca; |
|---|
| 895 | | - unsigned int i; |
|---|
| 929 | + struct cache *ca = c->cache; |
|---|
| 896 | 930 | |
|---|
| 897 | | - for_each_cache(ca, c, i) |
|---|
| 898 | | - wake_up_process(ca->alloc_thread); |
|---|
| 931 | + wake_up_process(ca->alloc_thread); |
|---|
| 899 | 932 | } |
|---|
| 900 | 933 | |
|---|
| 901 | 934 | static inline void closure_bio_submit(struct cache_set *c, |
|---|
| .. | .. |
|---|
| 908 | 941 | bio_endio(bio); |
|---|
| 909 | 942 | return; |
|---|
| 910 | 943 | } |
|---|
| 911 | | - generic_make_request(bio); |
|---|
| 944 | + submit_bio_noacct(bio); |
|---|
| 912 | 945 | } |
|---|
| 913 | 946 | |
|---|
| 914 | 947 | /* |
|---|
| .. | .. |
|---|
| 952 | 985 | |
|---|
| 953 | 986 | long bch_bucket_alloc(struct cache *ca, unsigned int reserve, bool wait); |
|---|
| 954 | 987 | int __bch_bucket_alloc_set(struct cache_set *c, unsigned int reserve, |
|---|
| 955 | | - struct bkey *k, int n, bool wait); |
|---|
| 988 | + struct bkey *k, bool wait); |
|---|
| 956 | 989 | int bch_bucket_alloc_set(struct cache_set *c, unsigned int reserve, |
|---|
| 957 | | - struct bkey *k, int n, bool wait); |
|---|
| 990 | + struct bkey *k, bool wait); |
|---|
| 958 | 991 | bool bch_alloc_sectors(struct cache_set *c, struct bkey *k, |
|---|
| 959 | 992 | unsigned int sectors, unsigned int write_point, |
|---|
| 960 | 993 | unsigned int write_prio, bool wait); |
|---|
| .. | .. |
|---|
| 968 | 1001 | |
|---|
| 969 | 1002 | extern struct workqueue_struct *bcache_wq; |
|---|
| 970 | 1003 | extern struct workqueue_struct *bch_journal_wq; |
|---|
| 1004 | +extern struct workqueue_struct *bch_flush_wq; |
|---|
| 971 | 1005 | extern struct mutex bch_register_lock; |
|---|
| 972 | 1006 | extern struct list_head bch_cache_sets; |
|---|
| 973 | 1007 | |
|---|
| .. | .. |
|---|
| 990 | 1024 | int bch_cached_dev_attach(struct cached_dev *dc, struct cache_set *c, |
|---|
| 991 | 1025 | uint8_t *set_uuid); |
|---|
| 992 | 1026 | void bch_cached_dev_detach(struct cached_dev *dc); |
|---|
| 993 | | -void bch_cached_dev_run(struct cached_dev *dc); |
|---|
| 1027 | +int bch_cached_dev_run(struct cached_dev *dc); |
|---|
| 994 | 1028 | void bcache_device_stop(struct bcache_device *d); |
|---|
| 995 | 1029 | |
|---|
| 996 | 1030 | void bch_cache_set_unregister(struct cache_set *c); |
|---|
| .. | .. |
|---|
| 1006 | 1040 | int bch_cache_allocator_start(struct cache *ca); |
|---|
| 1007 | 1041 | |
|---|
| 1008 | 1042 | void bch_debug_exit(void); |
|---|
| 1009 | | -void bch_debug_init(struct kobject *kobj); |
|---|
| 1043 | +void bch_debug_init(void); |
|---|
| 1010 | 1044 | void bch_request_exit(void); |
|---|
| 1011 | 1045 | int bch_request_init(void); |
|---|
| 1046 | +void bch_btree_exit(void); |
|---|
| 1047 | +int bch_btree_init(void); |
|---|
| 1012 | 1048 | |
|---|
| 1013 | 1049 | #endif /* _BCACHE_H */ |
|---|