hc
2024-05-10 61598093bbdd283a7edc367d900f223070ead8d2
kernel/mm/mempool.c
....@@ -58,11 +58,10 @@
5858 static void check_element(mempool_t *pool, void *element)
5959 {
6060 /* Mempools backed by slab allocator */
61
- if (pool->free == mempool_free_slab || pool->free == mempool_kfree)
61
+ if (pool->free == mempool_free_slab || pool->free == mempool_kfree) {
6262 __check_element(pool, element, ksize(element));
63
-
64
- /* Mempools backed by page allocator */
65
- if (pool->free == mempool_free_pages) {
63
+ } else if (pool->free == mempool_free_pages) {
64
+ /* Mempools backed by page allocator */
6665 int order = (int)(long)pool->pool_data;
6766 void *addr = kmap_atomic((struct page *)element);
6867
....@@ -82,11 +81,10 @@
8281 static void poison_element(mempool_t *pool, void *element)
8382 {
8483 /* Mempools backed by slab allocator */
85
- if (pool->alloc == mempool_alloc_slab || pool->alloc == mempool_kmalloc)
84
+ if (pool->alloc == mempool_alloc_slab || pool->alloc == mempool_kmalloc) {
8685 __poison_element(element, ksize(element));
87
-
88
- /* Mempools backed by page allocator */
89
- if (pool->alloc == mempool_alloc_pages) {
86
+ } else if (pool->alloc == mempool_alloc_pages) {
87
+ /* Mempools backed by page allocator */
9088 int order = (int)(long)pool->pool_data;
9189 void *addr = kmap_atomic((struct page *)element);
9290
....@@ -106,17 +104,19 @@
106104 static __always_inline void kasan_poison_element(mempool_t *pool, void *element)
107105 {
108106 if (pool->alloc == mempool_alloc_slab || pool->alloc == mempool_kmalloc)
109
- kasan_poison_kfree(element, _RET_IP_);
110
- if (pool->alloc == mempool_alloc_pages)
111
- kasan_free_pages(element, (unsigned long)pool->pool_data);
107
+ kasan_slab_free_mempool(element);
108
+ else if (pool->alloc == mempool_alloc_pages)
109
+ kasan_poison_pages(element, (unsigned long)pool->pool_data,
110
+ false);
112111 }
113112
114113 static void kasan_unpoison_element(mempool_t *pool, void *element)
115114 {
116115 if (pool->alloc == mempool_alloc_slab || pool->alloc == mempool_kmalloc)
117
- kasan_unpoison_slab(element);
118
- if (pool->alloc == mempool_alloc_pages)
119
- kasan_alloc_pages(element, (unsigned long)pool->pool_data);
116
+ kasan_unpoison_range(element, __ksize(element));
117
+ else if (pool->alloc == mempool_alloc_pages)
118
+ kasan_unpoison_pages(element, (unsigned long)pool->pool_data,
119
+ false);
120120 }
121121
122122 static __always_inline void add_element(mempool_t *pool, void *element)
....@@ -222,6 +222,8 @@
222222 *
223223 * Like mempool_create(), but initializes the pool in (i.e. embedded in another
224224 * structure).
225
+ *
226
+ * Return: %0 on success, negative error code otherwise.
225227 */
226228 int mempool_init(mempool_t *pool, int min_nr, mempool_alloc_t *alloc_fn,
227229 mempool_free_t *free_fn, void *pool_data)
....@@ -245,6 +247,8 @@
245247 * functions. This function might sleep. Both the alloc_fn() and the free_fn()
246248 * functions might sleep - as long as the mempool_alloc() function is not called
247249 * from IRQ contexts.
250
+ *
251
+ * Return: pointer to the created memory pool object or %NULL on error.
248252 */
249253 mempool_t *mempool_create(int min_nr, mempool_alloc_t *alloc_fn,
250254 mempool_free_t *free_fn, void *pool_data)
....@@ -289,6 +293,8 @@
289293 * Note, the caller must guarantee that no mempool_destroy is called
290294 * while this function is running. mempool_alloc() & mempool_free()
291295 * might be called (eg. from IRQ contexts) while this function executes.
296
+ *
297
+ * Return: %0 on success, negative error code otherwise.
292298 */
293299 int mempool_resize(mempool_t *pool, int new_min_nr)
294300 {
....@@ -363,6 +369,8 @@
363369 * *never* fails when called from process contexts. (it might
364370 * fail if called from an IRQ context.)
365371 * Note: using __GFP_ZERO is not supported.
372
+ *
373
+ * Return: pointer to the allocated element or %NULL on error.
366374 */
367375 void *mempool_alloc(mempool_t *pool, gfp_t gfp_mask)
368376 {
....@@ -481,7 +489,7 @@
481489 * ensures that there will be frees which return elements to the
482490 * pool waking up the waiters.
483491 */
484
- if (unlikely(pool->curr_nr < pool->min_nr)) {
492
+ if (unlikely(READ_ONCE(pool->curr_nr) < pool->min_nr)) {
485493 spin_lock_irqsave(&pool->lock, flags);
486494 if (likely(pool->curr_nr < pool->min_nr)) {
487495 add_element(pool, element);