| .. | .. |
|---|
| 1 | +// SPDX-License-Identifier: GPL-2.0-only |
|---|
| 1 | 2 | /* |
|---|
| 2 | | - * iteration_check.c: test races having to do with radix tree iteration |
|---|
| 3 | + * iteration_check.c: test races having to do with xarray iteration |
|---|
| 3 | 4 | * Copyright (c) 2016 Intel Corporation |
|---|
| 4 | 5 | * Author: Ross Zwisler <ross.zwisler@linux.intel.com> |
|---|
| 5 | | - * |
|---|
| 6 | | - * This program is free software; you can redistribute it and/or modify it |
|---|
| 7 | | - * under the terms and conditions of the GNU General Public License, |
|---|
| 8 | | - * version 2, as published by the Free Software Foundation. |
|---|
| 9 | | - * |
|---|
| 10 | | - * This program is distributed in the hope it will be useful, but WITHOUT |
|---|
| 11 | | - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
|---|
| 12 | | - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for |
|---|
| 13 | | - * more details. |
|---|
| 14 | 6 | */ |
|---|
| 15 | | -#include <linux/radix-tree.h> |
|---|
| 16 | 7 | #include <pthread.h> |
|---|
| 17 | 8 | #include "test.h" |
|---|
| 18 | 9 | |
|---|
| 19 | 10 | #define NUM_THREADS 5 |
|---|
| 20 | 11 | #define MAX_IDX 100 |
|---|
| 21 | | -#define TAG 0 |
|---|
| 22 | | -#define NEW_TAG 1 |
|---|
| 12 | +#define TAG XA_MARK_0 |
|---|
| 13 | +#define NEW_TAG XA_MARK_1 |
|---|
| 23 | 14 | |
|---|
| 24 | | -static pthread_mutex_t tree_lock = PTHREAD_MUTEX_INITIALIZER; |
|---|
| 25 | 15 | static pthread_t threads[NUM_THREADS]; |
|---|
| 26 | 16 | static unsigned int seeds[3]; |
|---|
| 27 | | -static RADIX_TREE(tree, GFP_KERNEL); |
|---|
| 17 | +static DEFINE_XARRAY(array); |
|---|
| 28 | 18 | static bool test_complete; |
|---|
| 29 | 19 | static int max_order; |
|---|
| 30 | 20 | |
|---|
| 31 | | -/* relentlessly fill the tree with tagged entries */ |
|---|
| 21 | +void my_item_insert(struct xarray *xa, unsigned long index) |
|---|
| 22 | +{ |
|---|
| 23 | + XA_STATE(xas, xa, index); |
|---|
| 24 | + struct item *item = item_create(index, 0); |
|---|
| 25 | + int order; |
|---|
| 26 | + |
|---|
| 27 | +retry: |
|---|
| 28 | + xas_lock(&xas); |
|---|
| 29 | + for (order = max_order; order >= 0; order--) { |
|---|
| 30 | + xas_set_order(&xas, index, order); |
|---|
| 31 | + item->order = order; |
|---|
| 32 | + if (xas_find_conflict(&xas)) |
|---|
| 33 | + continue; |
|---|
| 34 | + xas_store(&xas, item); |
|---|
| 35 | + xas_set_mark(&xas, TAG); |
|---|
| 36 | + break; |
|---|
| 37 | + } |
|---|
| 38 | + xas_unlock(&xas); |
|---|
| 39 | + if (xas_nomem(&xas, GFP_KERNEL)) |
|---|
| 40 | + goto retry; |
|---|
| 41 | + if (order < 0) |
|---|
| 42 | + free(item); |
|---|
| 43 | +} |
|---|
| 44 | + |
|---|
| 45 | +/* relentlessly fill the array with tagged entries */ |
|---|
| 32 | 46 | static void *add_entries_fn(void *arg) |
|---|
| 33 | 47 | { |
|---|
| 34 | 48 | rcu_register_thread(); |
|---|
| 35 | 49 | |
|---|
| 36 | 50 | while (!test_complete) { |
|---|
| 37 | 51 | unsigned long pgoff; |
|---|
| 38 | | - int order; |
|---|
| 39 | 52 | |
|---|
| 40 | 53 | for (pgoff = 0; pgoff < MAX_IDX; pgoff++) { |
|---|
| 41 | | - pthread_mutex_lock(&tree_lock); |
|---|
| 42 | | - for (order = max_order; order >= 0; order--) { |
|---|
| 43 | | - if (item_insert_order(&tree, pgoff, order) |
|---|
| 44 | | - == 0) { |
|---|
| 45 | | - item_tag_set(&tree, pgoff, TAG); |
|---|
| 46 | | - break; |
|---|
| 47 | | - } |
|---|
| 48 | | - } |
|---|
| 49 | | - pthread_mutex_unlock(&tree_lock); |
|---|
| 54 | + my_item_insert(&array, pgoff); |
|---|
| 50 | 55 | } |
|---|
| 51 | 56 | } |
|---|
| 52 | 57 | |
|---|
| .. | .. |
|---|
| 56 | 61 | } |
|---|
| 57 | 62 | |
|---|
| 58 | 63 | /* |
|---|
| 59 | | - * Iterate over the tagged entries, doing a radix_tree_iter_retry() as we find |
|---|
| 60 | | - * things that have been removed and randomly resetting our iteration to the |
|---|
| 61 | | - * next chunk with radix_tree_iter_resume(). Both radix_tree_iter_retry() and |
|---|
| 62 | | - * radix_tree_iter_resume() cause radix_tree_next_slot() to be called with a |
|---|
| 63 | | - * NULL 'slot' variable. |
|---|
| 64 | + * Iterate over tagged entries, retrying when we find ourselves in a deleted |
|---|
| 65 | + * node and randomly pausing the iteration. |
|---|
| 64 | 66 | */ |
|---|
| 65 | 67 | static void *tagged_iteration_fn(void *arg) |
|---|
| 66 | 68 | { |
|---|
| 67 | | - struct radix_tree_iter iter; |
|---|
| 68 | | - void **slot; |
|---|
| 69 | + XA_STATE(xas, &array, 0); |
|---|
| 70 | + void *entry; |
|---|
| 69 | 71 | |
|---|
| 70 | 72 | rcu_register_thread(); |
|---|
| 71 | 73 | |
|---|
| 72 | 74 | while (!test_complete) { |
|---|
| 75 | + xas_set(&xas, 0); |
|---|
| 73 | 76 | rcu_read_lock(); |
|---|
| 74 | | - radix_tree_for_each_tagged(slot, &tree, &iter, 0, TAG) { |
|---|
| 75 | | - void *entry = radix_tree_deref_slot(slot); |
|---|
| 76 | | - if (unlikely(!entry)) |
|---|
| 77 | + xas_for_each_marked(&xas, entry, ULONG_MAX, TAG) { |
|---|
| 78 | + if (xas_retry(&xas, entry)) |
|---|
| 77 | 79 | continue; |
|---|
| 78 | | - |
|---|
| 79 | | - if (radix_tree_deref_retry(entry)) { |
|---|
| 80 | | - slot = radix_tree_iter_retry(&iter); |
|---|
| 81 | | - continue; |
|---|
| 82 | | - } |
|---|
| 83 | 80 | |
|---|
| 84 | 81 | if (rand_r(&seeds[0]) % 50 == 0) { |
|---|
| 85 | | - slot = radix_tree_iter_resume(slot, &iter); |
|---|
| 82 | + xas_pause(&xas); |
|---|
| 86 | 83 | rcu_read_unlock(); |
|---|
| 87 | 84 | rcu_barrier(); |
|---|
| 88 | 85 | rcu_read_lock(); |
|---|
| .. | .. |
|---|
| 97 | 94 | } |
|---|
| 98 | 95 | |
|---|
| 99 | 96 | /* |
|---|
| 100 | | - * Iterate over the entries, doing a radix_tree_iter_retry() as we find things |
|---|
| 101 | | - * that have been removed and randomly resetting our iteration to the next |
|---|
| 102 | | - * chunk with radix_tree_iter_resume(). Both radix_tree_iter_retry() and |
|---|
| 103 | | - * radix_tree_iter_resume() cause radix_tree_next_slot() to be called with a |
|---|
| 104 | | - * NULL 'slot' variable. |
|---|
| 97 | + * Iterate over the entries, retrying when we find ourselves in a deleted |
|---|
| 98 | + * node and randomly pausing the iteration. |
|---|
| 105 | 99 | */ |
|---|
| 106 | 100 | static void *untagged_iteration_fn(void *arg) |
|---|
| 107 | 101 | { |
|---|
| 108 | | - struct radix_tree_iter iter; |
|---|
| 109 | | - void **slot; |
|---|
| 102 | + XA_STATE(xas, &array, 0); |
|---|
| 103 | + void *entry; |
|---|
| 110 | 104 | |
|---|
| 111 | 105 | rcu_register_thread(); |
|---|
| 112 | 106 | |
|---|
| 113 | 107 | while (!test_complete) { |
|---|
| 108 | + xas_set(&xas, 0); |
|---|
| 114 | 109 | rcu_read_lock(); |
|---|
| 115 | | - radix_tree_for_each_slot(slot, &tree, &iter, 0) { |
|---|
| 116 | | - void *entry = radix_tree_deref_slot(slot); |
|---|
| 117 | | - if (unlikely(!entry)) |
|---|
| 110 | + xas_for_each(&xas, entry, ULONG_MAX) { |
|---|
| 111 | + if (xas_retry(&xas, entry)) |
|---|
| 118 | 112 | continue; |
|---|
| 119 | | - |
|---|
| 120 | | - if (radix_tree_deref_retry(entry)) { |
|---|
| 121 | | - slot = radix_tree_iter_retry(&iter); |
|---|
| 122 | | - continue; |
|---|
| 123 | | - } |
|---|
| 124 | 113 | |
|---|
| 125 | 114 | if (rand_r(&seeds[1]) % 50 == 0) { |
|---|
| 126 | | - slot = radix_tree_iter_resume(slot, &iter); |
|---|
| 115 | + xas_pause(&xas); |
|---|
| 127 | 116 | rcu_read_unlock(); |
|---|
| 128 | 117 | rcu_barrier(); |
|---|
| 129 | 118 | rcu_read_lock(); |
|---|
| .. | .. |
|---|
| 138 | 127 | } |
|---|
| 139 | 128 | |
|---|
| 140 | 129 | /* |
|---|
| 141 | | - * Randomly remove entries to help induce radix_tree_iter_retry() calls in the |
|---|
| 130 | + * Randomly remove entries to help induce retries in the |
|---|
| 142 | 131 | * two iteration functions. |
|---|
| 143 | 132 | */ |
|---|
| 144 | 133 | static void *remove_entries_fn(void *arg) |
|---|
| .. | .. |
|---|
| 147 | 136 | |
|---|
| 148 | 137 | while (!test_complete) { |
|---|
| 149 | 138 | int pgoff; |
|---|
| 139 | + struct item *item; |
|---|
| 150 | 140 | |
|---|
| 151 | 141 | pgoff = rand_r(&seeds[2]) % MAX_IDX; |
|---|
| 152 | 142 | |
|---|
| 153 | | - pthread_mutex_lock(&tree_lock); |
|---|
| 154 | | - item_delete(&tree, pgoff); |
|---|
| 155 | | - pthread_mutex_unlock(&tree_lock); |
|---|
| 143 | + item = xa_erase(&array, pgoff); |
|---|
| 144 | + if (item) |
|---|
| 145 | + item_free(item, pgoff); |
|---|
| 156 | 146 | } |
|---|
| 157 | 147 | |
|---|
| 158 | 148 | rcu_unregister_thread(); |
|---|
| .. | .. |
|---|
| 165 | 155 | rcu_register_thread(); |
|---|
| 166 | 156 | |
|---|
| 167 | 157 | while (!test_complete) { |
|---|
| 168 | | - tag_tagged_items(&tree, &tree_lock, 0, MAX_IDX, 10, TAG, |
|---|
| 169 | | - NEW_TAG); |
|---|
| 158 | + tag_tagged_items(&array, 0, MAX_IDX, 10, TAG, NEW_TAG); |
|---|
| 170 | 159 | } |
|---|
| 171 | 160 | rcu_unregister_thread(); |
|---|
| 172 | 161 | return NULL; |
|---|
| .. | .. |
|---|
| 217 | 206 | } |
|---|
| 218 | 207 | } |
|---|
| 219 | 208 | |
|---|
| 220 | | - item_kill_tree(&tree); |
|---|
| 209 | + item_kill_tree(&array); |
|---|
| 221 | 210 | } |
|---|