.. | .. |
---|
8 | 8 | * Copyright (C) 2011 Google, Inc. |
---|
9 | 9 | */ |
---|
10 | 10 | |
---|
11 | | -#include <linux/module.h> |
---|
12 | 11 | #include <linux/freezer.h> |
---|
13 | 12 | #include <linux/list.h> |
---|
14 | 13 | #include <linux/slab.h> |
---|
| 14 | +#include <linux/spinlock.h> |
---|
15 | 15 | #include <linux/swap.h> |
---|
16 | 16 | #include <linux/sched/signal.h> |
---|
17 | 17 | #include "page_pool.h" |
---|
| 18 | + |
---|
| 19 | +struct dmabuf_page_pool_with_spinlock { |
---|
| 20 | + struct dmabuf_page_pool pool; |
---|
| 21 | + struct spinlock spinlock; |
---|
| 22 | +}; |
---|
18 | 23 | |
---|
19 | 24 | static LIST_HEAD(pool_list); |
---|
20 | 25 | static DEFINE_MUTEX(pool_list_lock); |
---|
.. | .. |
---|
36 | 41 | static void dmabuf_page_pool_add(struct dmabuf_page_pool *pool, struct page *page) |
---|
37 | 42 | { |
---|
38 | 43 | int index; |
---|
| 44 | + struct dmabuf_page_pool_with_spinlock *container_pool = |
---|
| 45 | + container_of(pool, struct dmabuf_page_pool_with_spinlock, pool); |
---|
39 | 46 | |
---|
40 | 47 | if (PageHighMem(page)) |
---|
41 | 48 | index = POOL_HIGHPAGE; |
---|
42 | 49 | else |
---|
43 | 50 | index = POOL_LOWPAGE; |
---|
44 | 51 | |
---|
45 | | - mutex_lock(&pool->mutex); |
---|
| 52 | + spin_lock(&container_pool->spinlock); |
---|
46 | 53 | list_add_tail(&page->lru, &pool->items[index]); |
---|
47 | 54 | pool->count[index]++; |
---|
| 55 | + spin_unlock(&container_pool->spinlock); |
---|
48 | 56 | mod_node_page_state(page_pgdat(page), NR_KERNEL_MISC_RECLAIMABLE, |
---|
49 | 57 | 1 << pool->order); |
---|
50 | | - mutex_unlock(&pool->mutex); |
---|
51 | 58 | } |
---|
52 | 59 | |
---|
53 | 60 | static struct page *dmabuf_page_pool_remove(struct dmabuf_page_pool *pool, int index) |
---|
54 | 61 | { |
---|
55 | 62 | struct page *page; |
---|
| 63 | + struct dmabuf_page_pool_with_spinlock *container_pool = |
---|
| 64 | + container_of(pool, struct dmabuf_page_pool_with_spinlock, pool); |
---|
56 | 65 | |
---|
57 | | - mutex_lock(&pool->mutex); |
---|
| 66 | + spin_lock(&container_pool->spinlock); |
---|
58 | 67 | page = list_first_entry_or_null(&pool->items[index], struct page, lru); |
---|
59 | 68 | if (page) { |
---|
60 | 69 | pool->count[index]--; |
---|
61 | 70 | list_del(&page->lru); |
---|
| 71 | + spin_unlock(&container_pool->spinlock); |
---|
62 | 72 | mod_node_page_state(page_pgdat(page), NR_KERNEL_MISC_RECLAIMABLE, |
---|
63 | 73 | -(1 << pool->order)); |
---|
| 74 | + goto out; |
---|
64 | 75 | } |
---|
65 | | - mutex_unlock(&pool->mutex); |
---|
| 76 | + spin_unlock(&container_pool->spinlock); |
---|
66 | 77 | |
---|
| 78 | +out: |
---|
67 | 79 | return page; |
---|
68 | 80 | } |
---|
69 | 81 | |
---|
.. | .. |
---|
114 | 126 | |
---|
115 | 127 | struct dmabuf_page_pool *dmabuf_page_pool_create(gfp_t gfp_mask, unsigned int order) |
---|
116 | 128 | { |
---|
117 | | - struct dmabuf_page_pool *pool = kmalloc(sizeof(*pool), GFP_KERNEL); |
---|
| 129 | + struct dmabuf_page_pool *pool; |
---|
| 130 | + struct dmabuf_page_pool_with_spinlock *container_pool = |
---|
| 131 | + kmalloc(sizeof(*container_pool), GFP_KERNEL); |
---|
118 | 132 | int i; |
---|
119 | 133 | |
---|
120 | | - if (!pool) |
---|
| 134 | + if (!container_pool) |
---|
121 | 135 | return NULL; |
---|
| 136 | + |
---|
| 137 | + spin_lock_init(&container_pool->spinlock); |
---|
| 138 | + pool = &container_pool->pool; |
---|
122 | 139 | |
---|
123 | 140 | for (i = 0; i < POOL_TYPE_SIZE; i++) { |
---|
124 | 141 | pool->count[i] = 0; |
---|
.. | .. |
---|
126 | 143 | } |
---|
127 | 144 | pool->gfp_mask = gfp_mask | __GFP_COMP; |
---|
128 | 145 | pool->order = order; |
---|
129 | | - mutex_init(&pool->mutex); |
---|
| 146 | + mutex_init(&pool->mutex); /* No longer used! */ |
---|
| 147 | + mutex_lock(&pool->mutex); /* Make sure anyone who attempts to acquire this hangs */ |
---|
130 | 148 | |
---|
131 | 149 | mutex_lock(&pool_list_lock); |
---|
132 | 150 | list_add(&pool->list, &pool_list); |
---|
.. | .. |
---|
139 | 157 | void dmabuf_page_pool_destroy(struct dmabuf_page_pool *pool) |
---|
140 | 158 | { |
---|
141 | 159 | struct page *page; |
---|
| 160 | + struct dmabuf_page_pool_with_spinlock *container_pool; |
---|
142 | 161 | int i; |
---|
143 | 162 | |
---|
144 | 163 | /* Remove us from the pool list */ |
---|
.. | .. |
---|
152 | 171 | dmabuf_page_pool_free_pages(pool, page); |
---|
153 | 172 | } |
---|
154 | 173 | |
---|
155 | | - kfree(pool); |
---|
| 174 | + container_pool = container_of(pool, struct dmabuf_page_pool_with_spinlock, pool); |
---|
| 175 | + kfree(container_pool); |
---|
156 | 176 | } |
---|
157 | 177 | EXPORT_SYMBOL_GPL(dmabuf_page_pool_destroy); |
---|
158 | 178 | |
---|
.. | .. |
---|
245 | 265 | return register_shrinker(&pool_shrinker); |
---|
246 | 266 | } |
---|
247 | 267 | module_init(dmabuf_page_pool_init_shrinker); |
---|
248 | | -MODULE_LICENSE("GPL"); |
---|
| 268 | +MODULE_LICENSE("GPL v2"); |
---|