.. | .. |
---|
16 | 16 | |
---|
17 | 17 | DEFINE_SPINLOCK(reuseport_lock); |
---|
18 | 18 | |
---|
19 | | -#define REUSEPORT_MIN_ID 1 |
---|
20 | 19 | static DEFINE_IDA(reuseport_ida); |
---|
21 | 20 | |
---|
22 | | -int reuseport_get_id(struct sock_reuseport *reuse) |
---|
| 21 | +void reuseport_has_conns_set(struct sock *sk) |
---|
23 | 22 | { |
---|
24 | | - int id; |
---|
| 23 | + struct sock_reuseport *reuse; |
---|
25 | 24 | |
---|
26 | | - if (reuse->reuseport_id) |
---|
27 | | - return reuse->reuseport_id; |
---|
| 25 | + if (!rcu_access_pointer(sk->sk_reuseport_cb)) |
---|
| 26 | + return; |
---|
28 | 27 | |
---|
29 | | - id = ida_simple_get(&reuseport_ida, REUSEPORT_MIN_ID, 0, |
---|
30 | | - /* Called under reuseport_lock */ |
---|
31 | | - GFP_ATOMIC); |
---|
32 | | - if (id < 0) |
---|
33 | | - return id; |
---|
34 | | - |
---|
35 | | - reuse->reuseport_id = id; |
---|
36 | | - |
---|
37 | | - return reuse->reuseport_id; |
---|
| 28 | + spin_lock_bh(&reuseport_lock); |
---|
| 29 | + reuse = rcu_dereference_protected(sk->sk_reuseport_cb, |
---|
| 30 | + lockdep_is_held(&reuseport_lock)); |
---|
| 31 | + if (likely(reuse)) |
---|
| 32 | + reuse->has_conns = 1; |
---|
| 33 | + spin_unlock_bh(&reuseport_lock); |
---|
38 | 34 | } |
---|
| 35 | +EXPORT_SYMBOL(reuseport_has_conns_set); |
---|
39 | 36 | |
---|
40 | 37 | static struct sock_reuseport *__reuseport_alloc(unsigned int max_socks) |
---|
41 | 38 | { |
---|
.. | .. |
---|
55 | 52 | int reuseport_alloc(struct sock *sk, bool bind_inany) |
---|
56 | 53 | { |
---|
57 | 54 | struct sock_reuseport *reuse; |
---|
| 55 | + int id, ret = 0; |
---|
58 | 56 | |
---|
59 | 57 | /* bh lock used since this function call may precede hlist lock in |
---|
60 | 58 | * soft irq of receive path or setsockopt from process context |
---|
.. | .. |
---|
78 | 76 | |
---|
79 | 77 | reuse = __reuseport_alloc(INIT_SOCKS); |
---|
80 | 78 | if (!reuse) { |
---|
81 | | - spin_unlock_bh(&reuseport_lock); |
---|
82 | | - return -ENOMEM; |
---|
| 79 | + ret = -ENOMEM; |
---|
| 80 | + goto out; |
---|
83 | 81 | } |
---|
84 | 82 | |
---|
| 83 | + id = ida_alloc(&reuseport_ida, GFP_ATOMIC); |
---|
| 84 | + if (id < 0) { |
---|
| 85 | + kfree(reuse); |
---|
| 86 | + ret = id; |
---|
| 87 | + goto out; |
---|
| 88 | + } |
---|
| 89 | + |
---|
| 90 | + reuse->reuseport_id = id; |
---|
85 | 91 | reuse->socks[0] = sk; |
---|
86 | 92 | reuse->num_socks = 1; |
---|
87 | 93 | reuse->bind_inany = bind_inany; |
---|
.. | .. |
---|
90 | 96 | out: |
---|
91 | 97 | spin_unlock_bh(&reuseport_lock); |
---|
92 | 98 | |
---|
93 | | - return 0; |
---|
| 99 | + return ret; |
---|
94 | 100 | } |
---|
95 | 101 | EXPORT_SYMBOL(reuseport_alloc); |
---|
96 | 102 | |
---|
.. | .. |
---|
107 | 113 | if (!more_reuse) |
---|
108 | 114 | return NULL; |
---|
109 | 115 | |
---|
110 | | - more_reuse->max_socks = more_socks_size; |
---|
111 | 116 | more_reuse->num_socks = reuse->num_socks; |
---|
112 | 117 | more_reuse->prog = reuse->prog; |
---|
113 | 118 | more_reuse->reuseport_id = reuse->reuseport_id; |
---|
.. | .. |
---|
136 | 141 | |
---|
137 | 142 | reuse = container_of(head, struct sock_reuseport, rcu); |
---|
138 | 143 | sk_reuseport_prog_free(rcu_dereference_protected(reuse->prog, 1)); |
---|
139 | | - if (reuse->reuseport_id) |
---|
140 | | - ida_simple_remove(&reuseport_ida, reuse->reuseport_id); |
---|
| 144 | + ida_free(&reuseport_ida, reuse->reuseport_id); |
---|
141 | 145 | kfree(reuse); |
---|
142 | 146 | } |
---|
143 | 147 | |
---|
.. | .. |
---|
145 | 149 | * reuseport_add_sock - Add a socket to the reuseport group of another. |
---|
146 | 150 | * @sk: New socket to add to the group. |
---|
147 | 151 | * @sk2: Socket belonging to the existing reuseport group. |
---|
| 152 | + * @bind_inany: Whether or not the group is bound to a local INANY address. |
---|
| 153 | + * |
---|
148 | 154 | * May return ENOMEM and not add socket to group under memory pressure. |
---|
149 | 155 | */ |
---|
150 | 156 | int reuseport_add_sock(struct sock *sk, struct sock *sk2, bool bind_inany) |
---|
.. | .. |
---|
188 | 194 | call_rcu(&old_reuse->rcu, reuseport_free_rcu); |
---|
189 | 195 | return 0; |
---|
190 | 196 | } |
---|
| 197 | +EXPORT_SYMBOL(reuseport_add_sock); |
---|
191 | 198 | |
---|
192 | 199 | void reuseport_detach_sock(struct sock *sk) |
---|
193 | 200 | { |
---|
.. | .. |
---|
198 | 205 | reuse = rcu_dereference_protected(sk->sk_reuseport_cb, |
---|
199 | 206 | lockdep_is_held(&reuseport_lock)); |
---|
200 | 207 | |
---|
201 | | - /* At least one of the sk in this reuseport group is added to |
---|
202 | | - * a bpf map. Notify the bpf side. The bpf map logic will |
---|
203 | | - * remove the sk if it is indeed added to a bpf map. |
---|
| 208 | + /* Notify the bpf side. The sk may be added to a sockarray |
---|
| 209 | + * map. If so, sockarray logic will remove it from the map. |
---|
| 210 | + * |
---|
| 211 | + * Other bpf map types that work with reuseport, like sockmap, |
---|
| 212 | + * don't need an explicit callback from here. They override sk |
---|
| 213 | + * unhash/close ops to remove the sk from the map before we |
---|
| 214 | + * get to this point. |
---|
204 | 215 | */ |
---|
205 | | - if (reuse->reuseport_id) |
---|
206 | | - bpf_sk_reuseport_detach(sk); |
---|
| 216 | + bpf_sk_reuseport_detach(sk); |
---|
207 | 217 | |
---|
208 | 218 | rcu_assign_pointer(sk->sk_reuseport_cb, NULL); |
---|
209 | 219 | |
---|
.. | .. |
---|
341 | 351 | return 0; |
---|
342 | 352 | } |
---|
343 | 353 | EXPORT_SYMBOL(reuseport_attach_prog); |
---|
| 354 | + |
---|
| 355 | +int reuseport_detach_prog(struct sock *sk) |
---|
| 356 | +{ |
---|
| 357 | + struct sock_reuseport *reuse; |
---|
| 358 | + struct bpf_prog *old_prog; |
---|
| 359 | + |
---|
| 360 | + if (!rcu_access_pointer(sk->sk_reuseport_cb)) |
---|
| 361 | + return sk->sk_reuseport ? -ENOENT : -EINVAL; |
---|
| 362 | + |
---|
| 363 | + old_prog = NULL; |
---|
| 364 | + spin_lock_bh(&reuseport_lock); |
---|
| 365 | + reuse = rcu_dereference_protected(sk->sk_reuseport_cb, |
---|
| 366 | + lockdep_is_held(&reuseport_lock)); |
---|
| 367 | + old_prog = rcu_replace_pointer(reuse->prog, old_prog, |
---|
| 368 | + lockdep_is_held(&reuseport_lock)); |
---|
| 369 | + spin_unlock_bh(&reuseport_lock); |
---|
| 370 | + |
---|
| 371 | + if (!old_prog) |
---|
| 372 | + return -ENOENT; |
---|
| 373 | + |
---|
| 374 | + sk_reuseport_prog_free(old_prog); |
---|
| 375 | + return 0; |
---|
| 376 | +} |
---|
| 377 | +EXPORT_SYMBOL(reuseport_detach_prog); |
---|