.. | .. |
---|
20 | 20 | /* The caller must hold the reuseport_lock */ |
---|
21 | 21 | void bpf_sk_reuseport_detach(struct sock *sk) |
---|
22 | 22 | { |
---|
23 | | - struct sock __rcu **socks; |
---|
| 23 | + uintptr_t sk_user_data; |
---|
24 | 24 | |
---|
25 | 25 | write_lock_bh(&sk->sk_callback_lock); |
---|
26 | | - socks = sk->sk_user_data; |
---|
27 | | - if (socks) { |
---|
| 26 | + sk_user_data = (uintptr_t)sk->sk_user_data; |
---|
| 27 | + if (sk_user_data & SK_USER_DATA_BPF) { |
---|
| 28 | + struct sock __rcu **socks; |
---|
| 29 | + |
---|
| 30 | + socks = (void *)(sk_user_data & SK_USER_DATA_PTRMASK); |
---|
28 | 31 | WRITE_ONCE(sk->sk_user_data, NULL); |
---|
29 | 32 | /* |
---|
30 | 33 | * Do not move this NULL assignment outside of |
---|
.. | .. |
---|
96 | 99 | struct sock *sk; |
---|
97 | 100 | u32 i; |
---|
98 | 101 | |
---|
99 | | - synchronize_rcu(); |
---|
100 | | - |
---|
101 | 102 | /* |
---|
102 | 103 | * ops->map_*_elem() will not be able to access this |
---|
103 | 104 | * array now. Hence, this function only races with |
---|
.. | .. |
---|
151 | 152 | { |
---|
152 | 153 | int err, numa_node = bpf_map_attr_numa_node(attr); |
---|
153 | 154 | struct reuseport_array *array; |
---|
154 | | - u64 cost, array_size; |
---|
| 155 | + struct bpf_map_memory mem; |
---|
| 156 | + u64 array_size; |
---|
155 | 157 | |
---|
156 | | - if (!capable(CAP_SYS_ADMIN)) |
---|
| 158 | + if (!bpf_capable()) |
---|
157 | 159 | return ERR_PTR(-EPERM); |
---|
158 | 160 | |
---|
159 | 161 | array_size = sizeof(*array); |
---|
160 | 162 | array_size += (u64)attr->max_entries * sizeof(struct sock *); |
---|
161 | 163 | |
---|
162 | | - /* make sure there is no u32 overflow later in round_up() */ |
---|
163 | | - cost = array_size; |
---|
164 | | - if (cost >= U32_MAX - PAGE_SIZE) |
---|
165 | | - return ERR_PTR(-ENOMEM); |
---|
166 | | - cost = round_up(cost, PAGE_SIZE) >> PAGE_SHIFT; |
---|
167 | | - |
---|
168 | | - err = bpf_map_precharge_memlock(cost); |
---|
| 164 | + err = bpf_map_charge_init(&mem, array_size); |
---|
169 | 165 | if (err) |
---|
170 | 166 | return ERR_PTR(err); |
---|
171 | 167 | |
---|
172 | 168 | /* allocate all map elements and zero-initialize them */ |
---|
173 | 169 | array = bpf_map_area_alloc(array_size, numa_node); |
---|
174 | | - if (!array) |
---|
| 170 | + if (!array) { |
---|
| 171 | + bpf_map_charge_finish(&mem); |
---|
175 | 172 | return ERR_PTR(-ENOMEM); |
---|
| 173 | + } |
---|
176 | 174 | |
---|
177 | 175 | /* copy mandatory map attributes */ |
---|
178 | 176 | bpf_map_init_from_attr(&array->map, attr); |
---|
179 | | - array->map.pages = cost; |
---|
| 177 | + bpf_map_charge_move(&array->map.memory, &mem); |
---|
180 | 178 | |
---|
181 | 179 | return &array->map; |
---|
182 | 180 | } |
---|
.. | .. |
---|
193 | 191 | rcu_read_lock(); |
---|
194 | 192 | sk = reuseport_array_lookup_elem(map, key); |
---|
195 | 193 | if (sk) { |
---|
196 | | - *(u64 *)value = sock_gen_cookie(sk); |
---|
| 194 | + *(u64 *)value = __sock_gen_cookie(sk); |
---|
197 | 195 | err = 0; |
---|
198 | 196 | } else { |
---|
199 | 197 | err = -ENOENT; |
---|
.. | .. |
---|
255 | 253 | struct sock *free_osk = NULL, *osk, *nsk; |
---|
256 | 254 | struct sock_reuseport *reuse; |
---|
257 | 255 | u32 index = *(u32 *)key; |
---|
| 256 | + uintptr_t sk_user_data; |
---|
258 | 257 | struct socket *socket; |
---|
259 | 258 | int err, fd; |
---|
260 | 259 | |
---|
.. | .. |
---|
308 | 307 | if (err) |
---|
309 | 308 | goto put_file_unlock; |
---|
310 | 309 | |
---|
311 | | - /* Ensure reuse->reuseport_id is set */ |
---|
312 | | - err = reuseport_get_id(reuse); |
---|
313 | | - if (err < 0) |
---|
314 | | - goto put_file_unlock; |
---|
315 | | - |
---|
316 | | - WRITE_ONCE(nsk->sk_user_data, &array->ptrs[index]); |
---|
| 310 | + sk_user_data = (uintptr_t)&array->ptrs[index] | SK_USER_DATA_NOCOPY | |
---|
| 311 | + SK_USER_DATA_BPF; |
---|
| 312 | + WRITE_ONCE(nsk->sk_user_data, (void *)sk_user_data); |
---|
317 | 313 | rcu_assign_pointer(array->ptrs[index], nsk); |
---|
318 | 314 | free_osk = osk; |
---|
319 | 315 | err = 0; |
---|
.. | .. |
---|
353 | 349 | return 0; |
---|
354 | 350 | } |
---|
355 | 351 | |
---|
| 352 | +static int reuseport_array_map_btf_id; |
---|
356 | 353 | const struct bpf_map_ops reuseport_array_ops = { |
---|
| 354 | + .map_meta_equal = bpf_map_meta_equal, |
---|
357 | 355 | .map_alloc_check = reuseport_array_alloc_check, |
---|
358 | 356 | .map_alloc = reuseport_array_alloc, |
---|
359 | 357 | .map_free = reuseport_array_free, |
---|
360 | 358 | .map_lookup_elem = reuseport_array_lookup_elem, |
---|
361 | 359 | .map_get_next_key = reuseport_array_get_next_key, |
---|
362 | 360 | .map_delete_elem = reuseport_array_delete_elem, |
---|
| 361 | + .map_btf_name = "reuseport_array", |
---|
| 362 | + .map_btf_id = &reuseport_array_map_btf_id, |
---|
363 | 363 | }; |
---|