| .. | .. |
|---|
| 62 | 62 | * IO workloads. |
|---|
| 63 | 63 | */ |
|---|
| 64 | 64 | |
|---|
| 65 | | -static int xen_blkif_max_buffer_pages = 1024; |
|---|
| 66 | | -module_param_named(max_buffer_pages, xen_blkif_max_buffer_pages, int, 0644); |
|---|
| 65 | +static int max_buffer_pages = 1024; |
|---|
| 66 | +module_param_named(max_buffer_pages, max_buffer_pages, int, 0644); |
|---|
| 67 | 67 | MODULE_PARM_DESC(max_buffer_pages, |
|---|
| 68 | 68 | "Maximum number of free pages to keep in each block backend buffer"); |
|---|
| 69 | 69 | |
|---|
| .. | .. |
|---|
| 78 | 78 | * algorithm. |
|---|
| 79 | 79 | */ |
|---|
| 80 | 80 | |
|---|
| 81 | | -static int xen_blkif_max_pgrants = 1056; |
|---|
| 82 | | -module_param_named(max_persistent_grants, xen_blkif_max_pgrants, int, 0644); |
|---|
| 81 | +static int max_pgrants = 1056; |
|---|
| 82 | +module_param_named(max_persistent_grants, max_pgrants, int, 0644); |
|---|
| 83 | 83 | MODULE_PARM_DESC(max_persistent_grants, |
|---|
| 84 | 84 | "Maximum number of grants to map persistently"); |
|---|
| 85 | 85 | |
|---|
| .. | .. |
|---|
| 88 | 88 | * use. The time is in seconds, 0 means indefinitely long. |
|---|
| 89 | 89 | */ |
|---|
| 90 | 90 | |
|---|
| 91 | | -static unsigned int xen_blkif_pgrant_timeout = 60; |
|---|
| 92 | | -module_param_named(persistent_grant_unused_seconds, xen_blkif_pgrant_timeout, |
|---|
| 91 | +static unsigned int pgrant_timeout = 60; |
|---|
| 92 | +module_param_named(persistent_grant_unused_seconds, pgrant_timeout, |
|---|
| 93 | 93 | uint, 0644); |
|---|
| 94 | 94 | MODULE_PARM_DESC(persistent_grant_unused_seconds, |
|---|
| 95 | 95 | "Time in seconds an unused persistent grant is allowed to " |
|---|
| .. | .. |
|---|
| 132 | 132 | |
|---|
| 133 | 133 | #define BLKBACK_INVALID_HANDLE (~0) |
|---|
| 134 | 134 | |
|---|
| 135 | | -/* Number of free pages to remove on each call to gnttab_free_pages */ |
|---|
| 136 | | -#define NUM_BATCH_FREE_PAGES 10 |
|---|
| 137 | | - |
|---|
| 138 | 135 | static inline bool persistent_gnt_timeout(struct persistent_gnt *persistent_gnt) |
|---|
| 139 | 136 | { |
|---|
| 140 | | - return xen_blkif_pgrant_timeout && |
|---|
| 141 | | - (jiffies - persistent_gnt->last_used >= |
|---|
| 142 | | - HZ * xen_blkif_pgrant_timeout); |
|---|
| 143 | | -} |
|---|
| 144 | | - |
|---|
| 145 | | -static inline int get_free_page(struct xen_blkif_ring *ring, struct page **page) |
|---|
| 146 | | -{ |
|---|
| 147 | | - unsigned long flags; |
|---|
| 148 | | - |
|---|
| 149 | | - spin_lock_irqsave(&ring->free_pages_lock, flags); |
|---|
| 150 | | - if (list_empty(&ring->free_pages)) { |
|---|
| 151 | | - BUG_ON(ring->free_pages_num != 0); |
|---|
| 152 | | - spin_unlock_irqrestore(&ring->free_pages_lock, flags); |
|---|
| 153 | | - return gnttab_alloc_pages(1, page); |
|---|
| 154 | | - } |
|---|
| 155 | | - BUG_ON(ring->free_pages_num == 0); |
|---|
| 156 | | - page[0] = list_first_entry(&ring->free_pages, struct page, lru); |
|---|
| 157 | | - list_del(&page[0]->lru); |
|---|
| 158 | | - ring->free_pages_num--; |
|---|
| 159 | | - spin_unlock_irqrestore(&ring->free_pages_lock, flags); |
|---|
| 160 | | - |
|---|
| 161 | | - return 0; |
|---|
| 162 | | -} |
|---|
| 163 | | - |
|---|
| 164 | | -static inline void put_free_pages(struct xen_blkif_ring *ring, struct page **page, |
|---|
| 165 | | - int num) |
|---|
| 166 | | -{ |
|---|
| 167 | | - unsigned long flags; |
|---|
| 168 | | - int i; |
|---|
| 169 | | - |
|---|
| 170 | | - spin_lock_irqsave(&ring->free_pages_lock, flags); |
|---|
| 171 | | - for (i = 0; i < num; i++) |
|---|
| 172 | | - list_add(&page[i]->lru, &ring->free_pages); |
|---|
| 173 | | - ring->free_pages_num += num; |
|---|
| 174 | | - spin_unlock_irqrestore(&ring->free_pages_lock, flags); |
|---|
| 175 | | -} |
|---|
| 176 | | - |
|---|
| 177 | | -static inline void shrink_free_pagepool(struct xen_blkif_ring *ring, int num) |
|---|
| 178 | | -{ |
|---|
| 179 | | - /* Remove requested pages in batches of NUM_BATCH_FREE_PAGES */ |
|---|
| 180 | | - struct page *page[NUM_BATCH_FREE_PAGES]; |
|---|
| 181 | | - unsigned int num_pages = 0; |
|---|
| 182 | | - unsigned long flags; |
|---|
| 183 | | - |
|---|
| 184 | | - spin_lock_irqsave(&ring->free_pages_lock, flags); |
|---|
| 185 | | - while (ring->free_pages_num > num) { |
|---|
| 186 | | - BUG_ON(list_empty(&ring->free_pages)); |
|---|
| 187 | | - page[num_pages] = list_first_entry(&ring->free_pages, |
|---|
| 188 | | - struct page, lru); |
|---|
| 189 | | - list_del(&page[num_pages]->lru); |
|---|
| 190 | | - ring->free_pages_num--; |
|---|
| 191 | | - if (++num_pages == NUM_BATCH_FREE_PAGES) { |
|---|
| 192 | | - spin_unlock_irqrestore(&ring->free_pages_lock, flags); |
|---|
| 193 | | - gnttab_free_pages(num_pages, page); |
|---|
| 194 | | - spin_lock_irqsave(&ring->free_pages_lock, flags); |
|---|
| 195 | | - num_pages = 0; |
|---|
| 196 | | - } |
|---|
| 197 | | - } |
|---|
| 198 | | - spin_unlock_irqrestore(&ring->free_pages_lock, flags); |
|---|
| 199 | | - if (num_pages != 0) |
|---|
| 200 | | - gnttab_free_pages(num_pages, page); |
|---|
| 137 | + return pgrant_timeout && (jiffies - persistent_gnt->last_used >= |
|---|
| 138 | + HZ * pgrant_timeout); |
|---|
| 201 | 139 | } |
|---|
| 202 | 140 | |
|---|
| 203 | 141 | #define vaddr(page) ((unsigned long)pfn_to_kaddr(page_to_pfn(page))) |
|---|
| .. | .. |
|---|
| 234 | 172 | struct persistent_gnt *this; |
|---|
| 235 | 173 | struct xen_blkif *blkif = ring->blkif; |
|---|
| 236 | 174 | |
|---|
| 237 | | - if (ring->persistent_gnt_c >= xen_blkif_max_pgrants) { |
|---|
| 175 | + if (ring->persistent_gnt_c >= max_pgrants) { |
|---|
| 238 | 176 | if (!blkif->vbd.overflow_max_grants) |
|---|
| 239 | 177 | blkif->vbd.overflow_max_grants = 1; |
|---|
| 240 | 178 | return -EBUSY; |
|---|
| .. | .. |
|---|
| 332 | 270 | unmap_data.count = segs_to_unmap; |
|---|
| 333 | 271 | BUG_ON(gnttab_unmap_refs_sync(&unmap_data)); |
|---|
| 334 | 272 | |
|---|
| 335 | | - put_free_pages(ring, pages, segs_to_unmap); |
|---|
| 273 | + gnttab_page_cache_put(&ring->free_pages, pages, |
|---|
| 274 | + segs_to_unmap); |
|---|
| 336 | 275 | segs_to_unmap = 0; |
|---|
| 337 | 276 | } |
|---|
| 338 | 277 | |
|---|
| .. | .. |
|---|
| 372 | 311 | if (++segs_to_unmap == BLKIF_MAX_SEGMENTS_PER_REQUEST) { |
|---|
| 373 | 312 | unmap_data.count = segs_to_unmap; |
|---|
| 374 | 313 | BUG_ON(gnttab_unmap_refs_sync(&unmap_data)); |
|---|
| 375 | | - put_free_pages(ring, pages, segs_to_unmap); |
|---|
| 314 | + gnttab_page_cache_put(&ring->free_pages, pages, |
|---|
| 315 | + segs_to_unmap); |
|---|
| 376 | 316 | segs_to_unmap = 0; |
|---|
| 377 | 317 | } |
|---|
| 378 | 318 | kfree(persistent_gnt); |
|---|
| .. | .. |
|---|
| 380 | 320 | if (segs_to_unmap > 0) { |
|---|
| 381 | 321 | unmap_data.count = segs_to_unmap; |
|---|
| 382 | 322 | BUG_ON(gnttab_unmap_refs_sync(&unmap_data)); |
|---|
| 383 | | - put_free_pages(ring, pages, segs_to_unmap); |
|---|
| 323 | + gnttab_page_cache_put(&ring->free_pages, pages, segs_to_unmap); |
|---|
| 384 | 324 | } |
|---|
| 385 | 325 | } |
|---|
| 386 | 326 | |
|---|
| .. | .. |
|---|
| 397 | 337 | goto out; |
|---|
| 398 | 338 | } |
|---|
| 399 | 339 | |
|---|
| 400 | | - if (ring->persistent_gnt_c < xen_blkif_max_pgrants || |
|---|
| 401 | | - (ring->persistent_gnt_c == xen_blkif_max_pgrants && |
|---|
| 340 | + if (ring->persistent_gnt_c < max_pgrants || |
|---|
| 341 | + (ring->persistent_gnt_c == max_pgrants && |
|---|
| 402 | 342 | !ring->blkif->vbd.overflow_max_grants)) { |
|---|
| 403 | 343 | num_clean = 0; |
|---|
| 404 | 344 | } else { |
|---|
| 405 | | - num_clean = (xen_blkif_max_pgrants / 100) * LRU_PERCENT_CLEAN; |
|---|
| 406 | | - num_clean = ring->persistent_gnt_c - xen_blkif_max_pgrants + |
|---|
| 407 | | - num_clean; |
|---|
| 345 | + num_clean = (max_pgrants / 100) * LRU_PERCENT_CLEAN; |
|---|
| 346 | + num_clean = ring->persistent_gnt_c - max_pgrants + num_clean; |
|---|
| 408 | 347 | num_clean = min(ring->persistent_gnt_c, num_clean); |
|---|
| 409 | 348 | pr_debug("Going to purge at least %u persistent grants\n", |
|---|
| 410 | 349 | num_clean); |
|---|
| .. | .. |
|---|
| 599 | 538 | current->comm, ring->st_oo_req, |
|---|
| 600 | 539 | ring->st_rd_req, ring->st_wr_req, |
|---|
| 601 | 540 | ring->st_f_req, ring->st_ds_req, |
|---|
| 602 | | - ring->persistent_gnt_c, |
|---|
| 603 | | - xen_blkif_max_pgrants); |
|---|
| 541 | + ring->persistent_gnt_c, max_pgrants); |
|---|
| 604 | 542 | ring->st_print = jiffies + msecs_to_jiffies(10 * 1000); |
|---|
| 605 | 543 | ring->st_rd_req = 0; |
|---|
| 606 | 544 | ring->st_wr_req = 0; |
|---|
| .. | .. |
|---|
| 665 | 603 | ring->next_lru = jiffies + msecs_to_jiffies(LRU_INTERVAL); |
|---|
| 666 | 604 | } |
|---|
| 667 | 605 | |
|---|
| 668 | | - /* Shrink if we have more than xen_blkif_max_buffer_pages */ |
|---|
| 669 | | - shrink_free_pagepool(ring, xen_blkif_max_buffer_pages); |
|---|
| 606 | + /* Shrink the free pages pool if it is too large. */ |
|---|
| 607 | + if (time_before(jiffies, blkif->buffer_squeeze_end)) |
|---|
| 608 | + gnttab_page_cache_shrink(&ring->free_pages, 0); |
|---|
| 609 | + else |
|---|
| 610 | + gnttab_page_cache_shrink(&ring->free_pages, |
|---|
| 611 | + max_buffer_pages); |
|---|
| 670 | 612 | |
|---|
| 671 | 613 | if (log_stats && time_after(jiffies, ring->st_print)) |
|---|
| 672 | 614 | print_stats(ring); |
|---|
| .. | .. |
|---|
| 697 | 639 | ring->persistent_gnt_c = 0; |
|---|
| 698 | 640 | |
|---|
| 699 | 641 | /* Since we are shutting down remove all pages from the buffer */ |
|---|
| 700 | | - shrink_free_pagepool(ring, 0 /* All */); |
|---|
| 642 | + gnttab_page_cache_shrink(&ring->free_pages, 0 /* All */); |
|---|
| 701 | 643 | } |
|---|
| 702 | 644 | |
|---|
| 703 | 645 | static unsigned int xen_blkbk_unmap_prepare( |
|---|
| .. | .. |
|---|
| 736 | 678 | but is this the best way to deal with this? */ |
|---|
| 737 | 679 | BUG_ON(result); |
|---|
| 738 | 680 | |
|---|
| 739 | | - put_free_pages(ring, data->pages, data->count); |
|---|
| 681 | + gnttab_page_cache_put(&ring->free_pages, data->pages, data->count); |
|---|
| 740 | 682 | make_response(ring, pending_req->id, |
|---|
| 741 | 683 | pending_req->operation, pending_req->status); |
|---|
| 742 | 684 | free_req(ring, pending_req); |
|---|
| .. | .. |
|---|
| 803 | 745 | if (invcount) { |
|---|
| 804 | 746 | ret = gnttab_unmap_refs(unmap, NULL, unmap_pages, invcount); |
|---|
| 805 | 747 | BUG_ON(ret); |
|---|
| 806 | | - put_free_pages(ring, unmap_pages, invcount); |
|---|
| 748 | + gnttab_page_cache_put(&ring->free_pages, unmap_pages, |
|---|
| 749 | + invcount); |
|---|
| 807 | 750 | } |
|---|
| 808 | 751 | pages += batch; |
|---|
| 809 | 752 | num -= batch; |
|---|
| .. | .. |
|---|
| 850 | 793 | pages[i]->page = persistent_gnt->page; |
|---|
| 851 | 794 | pages[i]->persistent_gnt = persistent_gnt; |
|---|
| 852 | 795 | } else { |
|---|
| 853 | | - if (get_free_page(ring, &pages[i]->page)) { |
|---|
| 854 | | - put_free_pages(ring, pages_to_gnt, segs_to_map); |
|---|
| 796 | + if (gnttab_page_cache_get(&ring->free_pages, |
|---|
| 797 | + &pages[i]->page)) { |
|---|
| 798 | + gnttab_page_cache_put(&ring->free_pages, |
|---|
| 799 | + pages_to_gnt, |
|---|
| 800 | + segs_to_map); |
|---|
| 855 | 801 | ret = -ENOMEM; |
|---|
| 856 | 802 | goto out; |
|---|
| 857 | 803 | } |
|---|
| .. | .. |
|---|
| 884 | 830 | BUG_ON(new_map_idx >= segs_to_map); |
|---|
| 885 | 831 | if (unlikely(map[new_map_idx].status != 0)) { |
|---|
| 886 | 832 | pr_debug("invalid buffer -- could not remap it\n"); |
|---|
| 887 | | - put_free_pages(ring, &pages[seg_idx]->page, 1); |
|---|
| 833 | + gnttab_page_cache_put(&ring->free_pages, |
|---|
| 834 | + &pages[seg_idx]->page, 1); |
|---|
| 888 | 835 | pages[seg_idx]->handle = BLKBACK_INVALID_HANDLE; |
|---|
| 889 | 836 | ret |= !ret; |
|---|
| 890 | 837 | goto next; |
|---|
| .. | .. |
|---|
| 894 | 841 | continue; |
|---|
| 895 | 842 | } |
|---|
| 896 | 843 | if (use_persistent_gnts && |
|---|
| 897 | | - ring->persistent_gnt_c < xen_blkif_max_pgrants) { |
|---|
| 844 | + ring->persistent_gnt_c < max_pgrants) { |
|---|
| 898 | 845 | /* |
|---|
| 899 | 846 | * We are using persistent grants, the grant is |
|---|
| 900 | 847 | * not mapped but we might have room for it. |
|---|
| .. | .. |
|---|
| 921 | 868 | pages[seg_idx]->persistent_gnt = persistent_gnt; |
|---|
| 922 | 869 | pr_debug("grant %u added to the tree of persistent grants, using %u/%u\n", |
|---|
| 923 | 870 | persistent_gnt->gnt, ring->persistent_gnt_c, |
|---|
| 924 | | - xen_blkif_max_pgrants); |
|---|
| 871 | + max_pgrants); |
|---|
| 925 | 872 | goto next; |
|---|
| 926 | 873 | } |
|---|
| 927 | 874 | if (use_persistent_gnts && !blkif->vbd.overflow_max_grants) { |
|---|
| .. | .. |
|---|
| 1274 | 1221 | break; |
|---|
| 1275 | 1222 | case BLKIF_OP_WRITE_BARRIER: |
|---|
| 1276 | 1223 | drain = true; |
|---|
| 1277 | | - /* fall through */ |
|---|
| 1224 | + fallthrough; |
|---|
| 1278 | 1225 | case BLKIF_OP_FLUSH_DISKCACHE: |
|---|
| 1279 | 1226 | ring->st_f_req++; |
|---|
| 1280 | 1227 | operation = REQ_OP_WRITE; |
|---|
| .. | .. |
|---|
| 1520 | 1467 | |
|---|
| 1521 | 1468 | module_init(xen_blkif_init); |
|---|
| 1522 | 1469 | |
|---|
| 1470 | +static void __exit xen_blkif_fini(void) |
|---|
| 1471 | +{ |
|---|
| 1472 | + xen_blkif_xenbus_fini(); |
|---|
| 1473 | + xen_blkif_interface_fini(); |
|---|
| 1474 | +} |
|---|
| 1475 | + |
|---|
| 1476 | +module_exit(xen_blkif_fini); |
|---|
| 1477 | + |
|---|
| 1523 | 1478 | MODULE_LICENSE("Dual BSD/GPL"); |
|---|
| 1524 | 1479 | MODULE_ALIAS("xen-backend:vbd"); |
|---|