| .. | .. |
|---|
| 25 | 25 | return radix_tree_tag_get(root, index, tag); |
|---|
| 26 | 26 | } |
|---|
| 27 | 27 | |
|---|
| 28 | | -int __item_insert(struct radix_tree_root *root, struct item *item) |
|---|
| 29 | | -{ |
|---|
| 30 | | - return __radix_tree_insert(root, item->index, item->order, item); |
|---|
| 31 | | -} |
|---|
| 32 | | - |
|---|
| 33 | 28 | struct item *item_create(unsigned long index, unsigned int order) |
|---|
| 34 | 29 | { |
|---|
| 35 | 30 | struct item *ret = malloc(sizeof(*ret)); |
|---|
| .. | .. |
|---|
| 39 | 34 | return ret; |
|---|
| 40 | 35 | } |
|---|
| 41 | 36 | |
|---|
| 42 | | -int item_insert_order(struct radix_tree_root *root, unsigned long index, |
|---|
| 43 | | - unsigned order) |
|---|
| 37 | +int item_insert(struct radix_tree_root *root, unsigned long index) |
|---|
| 44 | 38 | { |
|---|
| 45 | | - struct item *item = item_create(index, order); |
|---|
| 46 | | - int err = __item_insert(root, item); |
|---|
| 39 | + struct item *item = item_create(index, 0); |
|---|
| 40 | + int err = radix_tree_insert(root, item->index, item); |
|---|
| 47 | 41 | if (err) |
|---|
| 48 | 42 | free(item); |
|---|
| 49 | 43 | return err; |
|---|
| 50 | | -} |
|---|
| 51 | | - |
|---|
| 52 | | -int item_insert(struct radix_tree_root *root, unsigned long index) |
|---|
| 53 | | -{ |
|---|
| 54 | | - return item_insert_order(root, index, 0); |
|---|
| 55 | 44 | } |
|---|
| 56 | 45 | |
|---|
| 57 | 46 | void item_sanity(struct item *item, unsigned long index) |
|---|
| .. | .. |
|---|
| 63 | 52 | assert((item->index | mask) == (index | mask)); |
|---|
| 64 | 53 | } |
|---|
| 65 | 54 | |
|---|
| 55 | +void item_free(struct item *item, unsigned long index) |
|---|
| 56 | +{ |
|---|
| 57 | + item_sanity(item, index); |
|---|
| 58 | + free(item); |
|---|
| 59 | +} |
|---|
| 60 | + |
|---|
| 66 | 61 | int item_delete(struct radix_tree_root *root, unsigned long index) |
|---|
| 67 | 62 | { |
|---|
| 68 | 63 | struct item *item = radix_tree_delete(root, index); |
|---|
| 69 | 64 | |
|---|
| 70 | | - if (item) { |
|---|
| 71 | | - item_sanity(item, index); |
|---|
| 72 | | - free(item); |
|---|
| 73 | | - return 1; |
|---|
| 74 | | - } |
|---|
| 75 | | - return 0; |
|---|
| 65 | + if (!item) |
|---|
| 66 | + return 0; |
|---|
| 67 | + |
|---|
| 68 | + item_free(item, index); |
|---|
| 69 | + return 1; |
|---|
| 76 | 70 | } |
|---|
| 77 | 71 | |
|---|
| 78 | 72 | static void item_free_rcu(struct rcu_head *head) |
|---|
| .. | .. |
|---|
| 82 | 76 | free(item); |
|---|
| 83 | 77 | } |
|---|
| 84 | 78 | |
|---|
| 85 | | -int item_delete_rcu(struct radix_tree_root *root, unsigned long index) |
|---|
| 79 | +int item_delete_rcu(struct xarray *xa, unsigned long index) |
|---|
| 86 | 80 | { |
|---|
| 87 | | - struct item *item = radix_tree_delete(root, index); |
|---|
| 81 | + struct item *item = xa_erase(xa, index); |
|---|
| 88 | 82 | |
|---|
| 89 | 83 | if (item) { |
|---|
| 90 | 84 | item_sanity(item, index); |
|---|
| .. | .. |
|---|
| 176 | 170 | } |
|---|
| 177 | 171 | |
|---|
| 178 | 172 | /* Use the same pattern as tag_pages_for_writeback() in mm/page-writeback.c */ |
|---|
| 179 | | -int tag_tagged_items(struct radix_tree_root *root, pthread_mutex_t *lock, |
|---|
| 180 | | - unsigned long start, unsigned long end, unsigned batch, |
|---|
| 181 | | - unsigned iftag, unsigned thentag) |
|---|
| 173 | +int tag_tagged_items(struct xarray *xa, unsigned long start, unsigned long end, |
|---|
| 174 | + unsigned batch, xa_mark_t iftag, xa_mark_t thentag) |
|---|
| 182 | 175 | { |
|---|
| 183 | | - unsigned long tagged = 0; |
|---|
| 184 | | - struct radix_tree_iter iter; |
|---|
| 185 | | - void **slot; |
|---|
| 176 | + XA_STATE(xas, xa, start); |
|---|
| 177 | + unsigned int tagged = 0; |
|---|
| 178 | + struct item *item; |
|---|
| 186 | 179 | |
|---|
| 187 | 180 | if (batch == 0) |
|---|
| 188 | 181 | batch = 1; |
|---|
| 189 | 182 | |
|---|
| 190 | | - if (lock) |
|---|
| 191 | | - pthread_mutex_lock(lock); |
|---|
| 192 | | - radix_tree_for_each_tagged(slot, root, &iter, start, iftag) { |
|---|
| 193 | | - if (iter.index > end) |
|---|
| 194 | | - break; |
|---|
| 195 | | - radix_tree_iter_tag_set(root, &iter, thentag); |
|---|
| 196 | | - tagged++; |
|---|
| 197 | | - if ((tagged % batch) != 0) |
|---|
| 183 | + xas_lock_irq(&xas); |
|---|
| 184 | + xas_for_each_marked(&xas, item, end, iftag) { |
|---|
| 185 | + xas_set_mark(&xas, thentag); |
|---|
| 186 | + if (++tagged % batch) |
|---|
| 198 | 187 | continue; |
|---|
| 199 | | - slot = radix_tree_iter_resume(slot, &iter); |
|---|
| 200 | | - if (lock) { |
|---|
| 201 | | - pthread_mutex_unlock(lock); |
|---|
| 202 | | - rcu_barrier(); |
|---|
| 203 | | - pthread_mutex_lock(lock); |
|---|
| 204 | | - } |
|---|
| 188 | + |
|---|
| 189 | + xas_pause(&xas); |
|---|
| 190 | + xas_unlock_irq(&xas); |
|---|
| 191 | + rcu_barrier(); |
|---|
| 192 | + xas_lock_irq(&xas); |
|---|
| 205 | 193 | } |
|---|
| 206 | | - if (lock) |
|---|
| 207 | | - pthread_mutex_unlock(lock); |
|---|
| 194 | + xas_unlock_irq(&xas); |
|---|
| 208 | 195 | |
|---|
| 209 | 196 | return tagged; |
|---|
| 210 | | -} |
|---|
| 211 | | - |
|---|
| 212 | | -/* Use the same pattern as find_swap_entry() in mm/shmem.c */ |
|---|
| 213 | | -unsigned long find_item(struct radix_tree_root *root, void *item) |
|---|
| 214 | | -{ |
|---|
| 215 | | - struct radix_tree_iter iter; |
|---|
| 216 | | - void **slot; |
|---|
| 217 | | - unsigned long found = -1; |
|---|
| 218 | | - unsigned long checked = 0; |
|---|
| 219 | | - |
|---|
| 220 | | - radix_tree_for_each_slot(slot, root, &iter, 0) { |
|---|
| 221 | | - if (*slot == item) { |
|---|
| 222 | | - found = iter.index; |
|---|
| 223 | | - break; |
|---|
| 224 | | - } |
|---|
| 225 | | - checked++; |
|---|
| 226 | | - if ((checked % 4) != 0) |
|---|
| 227 | | - continue; |
|---|
| 228 | | - slot = radix_tree_iter_resume(slot, &iter); |
|---|
| 229 | | - } |
|---|
| 230 | | - |
|---|
| 231 | | - return found; |
|---|
| 232 | 197 | } |
|---|
| 233 | 198 | |
|---|
| 234 | 199 | static int verify_node(struct radix_tree_node *slot, unsigned int tag, |
|---|
| .. | .. |
|---|
| 281 | 246 | |
|---|
| 282 | 247 | void verify_tag_consistency(struct radix_tree_root *root, unsigned int tag) |
|---|
| 283 | 248 | { |
|---|
| 284 | | - struct radix_tree_node *node = root->rnode; |
|---|
| 249 | + struct radix_tree_node *node = root->xa_head; |
|---|
| 285 | 250 | if (!radix_tree_is_internal_node(node)) |
|---|
| 286 | 251 | return; |
|---|
| 287 | 252 | verify_node(node, tag, !!root_tag_get(root, tag)); |
|---|
| 288 | 253 | } |
|---|
| 289 | 254 | |
|---|
| 290 | | -void item_kill_tree(struct radix_tree_root *root) |
|---|
| 255 | +void item_kill_tree(struct xarray *xa) |
|---|
| 291 | 256 | { |
|---|
| 292 | | - struct radix_tree_iter iter; |
|---|
| 293 | | - void **slot; |
|---|
| 294 | | - struct item *items[32]; |
|---|
| 295 | | - int nfound; |
|---|
| 257 | + XA_STATE(xas, xa, 0); |
|---|
| 258 | + void *entry; |
|---|
| 296 | 259 | |
|---|
| 297 | | - radix_tree_for_each_slot(slot, root, &iter, 0) { |
|---|
| 298 | | - if (radix_tree_exceptional_entry(*slot)) |
|---|
| 299 | | - radix_tree_delete(root, iter.index); |
|---|
| 300 | | - } |
|---|
| 301 | | - |
|---|
| 302 | | - while ((nfound = radix_tree_gang_lookup(root, (void **)items, 0, 32))) { |
|---|
| 303 | | - int i; |
|---|
| 304 | | - |
|---|
| 305 | | - for (i = 0; i < nfound; i++) { |
|---|
| 306 | | - void *ret; |
|---|
| 307 | | - |
|---|
| 308 | | - ret = radix_tree_delete(root, items[i]->index); |
|---|
| 309 | | - assert(ret == items[i]); |
|---|
| 310 | | - free(items[i]); |
|---|
| 260 | + xas_for_each(&xas, entry, ULONG_MAX) { |
|---|
| 261 | + if (!xa_is_value(entry)) { |
|---|
| 262 | + item_free(entry, xas.xa_index); |
|---|
| 311 | 263 | } |
|---|
| 264 | + xas_store(&xas, NULL); |
|---|
| 312 | 265 | } |
|---|
| 313 | | - assert(radix_tree_gang_lookup(root, (void **)items, 0, 32) == 0); |
|---|
| 314 | | - assert(root->rnode == NULL); |
|---|
| 266 | + |
|---|
| 267 | + assert(xa_empty(xa)); |
|---|
| 315 | 268 | } |
|---|
| 316 | 269 | |
|---|
| 317 | 270 | void tree_verify_min_height(struct radix_tree_root *root, int maxindex) |
|---|
| 318 | 271 | { |
|---|
| 319 | 272 | unsigned shift; |
|---|
| 320 | | - struct radix_tree_node *node = root->rnode; |
|---|
| 273 | + struct radix_tree_node *node = root->xa_head; |
|---|
| 321 | 274 | if (!radix_tree_is_internal_node(node)) { |
|---|
| 322 | 275 | assert(maxindex == 0); |
|---|
| 323 | 276 | return; |
|---|