.. | .. |
---|
41 | 41 | /* bpf_lru_node helpers */ |
---|
42 | 42 | static bool bpf_lru_node_is_ref(const struct bpf_lru_node *node) |
---|
43 | 43 | { |
---|
44 | | - return node->ref; |
---|
| 44 | + return READ_ONCE(node->ref); |
---|
| 45 | +} |
---|
| 46 | + |
---|
| 47 | +static void bpf_lru_node_clear_ref(struct bpf_lru_node *node) |
---|
| 48 | +{ |
---|
| 49 | + WRITE_ONCE(node->ref, 0); |
---|
45 | 50 | } |
---|
46 | 51 | |
---|
47 | 52 | static void bpf_lru_list_count_inc(struct bpf_lru_list *l, |
---|
.. | .. |
---|
89 | 94 | |
---|
90 | 95 | bpf_lru_list_count_inc(l, tgt_type); |
---|
91 | 96 | node->type = tgt_type; |
---|
92 | | - node->ref = 0; |
---|
| 97 | + bpf_lru_node_clear_ref(node); |
---|
93 | 98 | list_move(&node->list, &l->lists[tgt_type]); |
---|
94 | 99 | } |
---|
95 | 100 | |
---|
.. | .. |
---|
110 | 115 | bpf_lru_list_count_inc(l, tgt_type); |
---|
111 | 116 | node->type = tgt_type; |
---|
112 | 117 | } |
---|
113 | | - node->ref = 0; |
---|
| 118 | + bpf_lru_node_clear_ref(node); |
---|
114 | 119 | |
---|
115 | 120 | /* If the moving node is the next_inactive_rotation candidate, |
---|
116 | 121 | * move the next_inactive_rotation pointer also. |
---|
.. | .. |
---|
353 | 358 | *(u32 *)((void *)node + lru->hash_offset) = hash; |
---|
354 | 359 | node->cpu = cpu; |
---|
355 | 360 | node->type = BPF_LRU_LOCAL_LIST_T_PENDING; |
---|
356 | | - node->ref = 0; |
---|
| 361 | + bpf_lru_node_clear_ref(node); |
---|
357 | 362 | list_add(&node->list, local_pending_list(loc_l)); |
---|
358 | 363 | } |
---|
359 | 364 | |
---|
.. | .. |
---|
419 | 424 | if (!list_empty(free_list)) { |
---|
420 | 425 | node = list_first_entry(free_list, struct bpf_lru_node, list); |
---|
421 | 426 | *(u32 *)((void *)node + lru->hash_offset) = hash; |
---|
422 | | - node->ref = 0; |
---|
| 427 | + bpf_lru_node_clear_ref(node); |
---|
423 | 428 | __bpf_lru_node_move(l, node, BPF_LRU_LIST_T_INACTIVE); |
---|
424 | 429 | } |
---|
425 | 430 | |
---|
.. | .. |
---|
522 | 527 | } |
---|
523 | 528 | |
---|
524 | 529 | node->type = BPF_LRU_LOCAL_LIST_T_FREE; |
---|
525 | | - node->ref = 0; |
---|
| 530 | + bpf_lru_node_clear_ref(node); |
---|
526 | 531 | list_move(&node->list, local_free_list(loc_l)); |
---|
527 | 532 | |
---|
528 | 533 | raw_spin_unlock_irqrestore(&loc_l->lock, flags); |
---|
.. | .. |
---|
568 | 573 | |
---|
569 | 574 | node = (struct bpf_lru_node *)(buf + node_offset); |
---|
570 | 575 | node->type = BPF_LRU_LIST_T_FREE; |
---|
571 | | - node->ref = 0; |
---|
| 576 | + bpf_lru_node_clear_ref(node); |
---|
572 | 577 | list_add(&node->list, &l->lists[BPF_LRU_LIST_T_FREE]); |
---|
573 | 578 | buf += elem_size; |
---|
574 | 579 | } |
---|
.. | .. |
---|
594 | 599 | node = (struct bpf_lru_node *)(buf + node_offset); |
---|
595 | 600 | node->cpu = cpu; |
---|
596 | 601 | node->type = BPF_LRU_LIST_T_FREE; |
---|
597 | | - node->ref = 0; |
---|
| 602 | + bpf_lru_node_clear_ref(node); |
---|
598 | 603 | list_add(&node->list, &l->lists[BPF_LRU_LIST_T_FREE]); |
---|
599 | 604 | i++; |
---|
600 | 605 | buf += elem_size; |
---|