| .. | .. |
|---|
| 41 | 41 | /* bpf_lru_node helpers */ |
|---|
| 42 | 42 | static bool bpf_lru_node_is_ref(const struct bpf_lru_node *node) |
|---|
| 43 | 43 | { |
|---|
| 44 | | - return node->ref; |
|---|
| 44 | + return READ_ONCE(node->ref); |
|---|
| 45 | +} |
|---|
| 46 | + |
|---|
| 47 | +static void bpf_lru_node_clear_ref(struct bpf_lru_node *node) |
|---|
| 48 | +{ |
|---|
| 49 | + WRITE_ONCE(node->ref, 0); |
|---|
| 45 | 50 | } |
|---|
| 46 | 51 | |
|---|
| 47 | 52 | static void bpf_lru_list_count_inc(struct bpf_lru_list *l, |
|---|
| .. | .. |
|---|
| 89 | 94 | |
|---|
| 90 | 95 | bpf_lru_list_count_inc(l, tgt_type); |
|---|
| 91 | 96 | node->type = tgt_type; |
|---|
| 92 | | - node->ref = 0; |
|---|
| 97 | + bpf_lru_node_clear_ref(node); |
|---|
| 93 | 98 | list_move(&node->list, &l->lists[tgt_type]); |
|---|
| 94 | 99 | } |
|---|
| 95 | 100 | |
|---|
| .. | .. |
|---|
| 110 | 115 | bpf_lru_list_count_inc(l, tgt_type); |
|---|
| 111 | 116 | node->type = tgt_type; |
|---|
| 112 | 117 | } |
|---|
| 113 | | - node->ref = 0; |
|---|
| 118 | + bpf_lru_node_clear_ref(node); |
|---|
| 114 | 119 | |
|---|
| 115 | 120 | /* If the moving node is the next_inactive_rotation candidate, |
|---|
| 116 | 121 | * move the next_inactive_rotation pointer also. |
|---|
| .. | .. |
|---|
| 353 | 358 | *(u32 *)((void *)node + lru->hash_offset) = hash; |
|---|
| 354 | 359 | node->cpu = cpu; |
|---|
| 355 | 360 | node->type = BPF_LRU_LOCAL_LIST_T_PENDING; |
|---|
| 356 | | - node->ref = 0; |
|---|
| 361 | + bpf_lru_node_clear_ref(node); |
|---|
| 357 | 362 | list_add(&node->list, local_pending_list(loc_l)); |
|---|
| 358 | 363 | } |
|---|
| 359 | 364 | |
|---|
| .. | .. |
|---|
| 419 | 424 | if (!list_empty(free_list)) { |
|---|
| 420 | 425 | node = list_first_entry(free_list, struct bpf_lru_node, list); |
|---|
| 421 | 426 | *(u32 *)((void *)node + lru->hash_offset) = hash; |
|---|
| 422 | | - node->ref = 0; |
|---|
| 427 | + bpf_lru_node_clear_ref(node); |
|---|
| 423 | 428 | __bpf_lru_node_move(l, node, BPF_LRU_LIST_T_INACTIVE); |
|---|
| 424 | 429 | } |
|---|
| 425 | 430 | |
|---|
| .. | .. |
|---|
| 522 | 527 | } |
|---|
| 523 | 528 | |
|---|
| 524 | 529 | node->type = BPF_LRU_LOCAL_LIST_T_FREE; |
|---|
| 525 | | - node->ref = 0; |
|---|
| 530 | + bpf_lru_node_clear_ref(node); |
|---|
| 526 | 531 | list_move(&node->list, local_free_list(loc_l)); |
|---|
| 527 | 532 | |
|---|
| 528 | 533 | raw_spin_unlock_irqrestore(&loc_l->lock, flags); |
|---|
| .. | .. |
|---|
| 568 | 573 | |
|---|
| 569 | 574 | node = (struct bpf_lru_node *)(buf + node_offset); |
|---|
| 570 | 575 | node->type = BPF_LRU_LIST_T_FREE; |
|---|
| 571 | | - node->ref = 0; |
|---|
| 576 | + bpf_lru_node_clear_ref(node); |
|---|
| 572 | 577 | list_add(&node->list, &l->lists[BPF_LRU_LIST_T_FREE]); |
|---|
| 573 | 578 | buf += elem_size; |
|---|
| 574 | 579 | } |
|---|
| .. | .. |
|---|
| 594 | 599 | node = (struct bpf_lru_node *)(buf + node_offset); |
|---|
| 595 | 600 | node->cpu = cpu; |
|---|
| 596 | 601 | node->type = BPF_LRU_LIST_T_FREE; |
|---|
| 597 | | - node->ref = 0; |
|---|
| 602 | + bpf_lru_node_clear_ref(node); |
|---|
| 598 | 603 | list_add(&node->list, &l->lists[BPF_LRU_LIST_T_FREE]); |
|---|
| 599 | 604 | i++; |
|---|
| 600 | 605 | buf += elem_size; |
|---|