hc
2024-05-10 748e4f3d702def1a4bff191e0cf93b6a05340f01
kernel/kernel/bpf/reuseport_array.c
....@@ -20,11 +20,14 @@
2020 /* The caller must hold the reuseport_lock */
2121 void bpf_sk_reuseport_detach(struct sock *sk)
2222 {
23
- struct sock __rcu **socks;
23
+ uintptr_t sk_user_data;
2424
2525 write_lock_bh(&sk->sk_callback_lock);
26
- socks = sk->sk_user_data;
27
- if (socks) {
26
+ sk_user_data = (uintptr_t)sk->sk_user_data;
27
+ if (sk_user_data & SK_USER_DATA_BPF) {
28
+ struct sock __rcu **socks;
29
+
30
+ socks = (void *)(sk_user_data & SK_USER_DATA_PTRMASK);
2831 WRITE_ONCE(sk->sk_user_data, NULL);
2932 /*
3033 * Do not move this NULL assignment outside of
....@@ -96,8 +99,6 @@
9699 struct sock *sk;
97100 u32 i;
98101
99
- synchronize_rcu();
100
-
101102 /*
102103 * ops->map_*_elem() will not be able to access this
103104 * array now. Hence, this function only races with
....@@ -151,32 +152,29 @@
151152 {
152153 int err, numa_node = bpf_map_attr_numa_node(attr);
153154 struct reuseport_array *array;
154
- u64 cost, array_size;
155
+ struct bpf_map_memory mem;
156
+ u64 array_size;
155157
156
- if (!capable(CAP_SYS_ADMIN))
158
+ if (!bpf_capable())
157159 return ERR_PTR(-EPERM);
158160
159161 array_size = sizeof(*array);
160162 array_size += (u64)attr->max_entries * sizeof(struct sock *);
161163
162
- /* make sure there is no u32 overflow later in round_up() */
163
- cost = array_size;
164
- if (cost >= U32_MAX - PAGE_SIZE)
165
- return ERR_PTR(-ENOMEM);
166
- cost = round_up(cost, PAGE_SIZE) >> PAGE_SHIFT;
167
-
168
- err = bpf_map_precharge_memlock(cost);
164
+ err = bpf_map_charge_init(&mem, array_size);
169165 if (err)
170166 return ERR_PTR(err);
171167
172168 /* allocate all map elements and zero-initialize them */
173169 array = bpf_map_area_alloc(array_size, numa_node);
174
- if (!array)
170
+ if (!array) {
171
+ bpf_map_charge_finish(&mem);
175172 return ERR_PTR(-ENOMEM);
173
+ }
176174
177175 /* copy mandatory map attributes */
178176 bpf_map_init_from_attr(&array->map, attr);
179
- array->map.pages = cost;
177
+ bpf_map_charge_move(&array->map.memory, &mem);
180178
181179 return &array->map;
182180 }
....@@ -193,7 +191,7 @@
193191 rcu_read_lock();
194192 sk = reuseport_array_lookup_elem(map, key);
195193 if (sk) {
196
- *(u64 *)value = sock_gen_cookie(sk);
194
+ *(u64 *)value = __sock_gen_cookie(sk);
197195 err = 0;
198196 } else {
199197 err = -ENOENT;
....@@ -255,6 +253,7 @@
255253 struct sock *free_osk = NULL, *osk, *nsk;
256254 struct sock_reuseport *reuse;
257255 u32 index = *(u32 *)key;
256
+ uintptr_t sk_user_data;
258257 struct socket *socket;
259258 int err, fd;
260259
....@@ -308,12 +307,9 @@
308307 if (err)
309308 goto put_file_unlock;
310309
311
- /* Ensure reuse->reuseport_id is set */
312
- err = reuseport_get_id(reuse);
313
- if (err < 0)
314
- goto put_file_unlock;
315
-
316
- WRITE_ONCE(nsk->sk_user_data, &array->ptrs[index]);
310
+ sk_user_data = (uintptr_t)&array->ptrs[index] | SK_USER_DATA_NOCOPY |
311
+ SK_USER_DATA_BPF;
312
+ WRITE_ONCE(nsk->sk_user_data, (void *)sk_user_data);
317313 rcu_assign_pointer(array->ptrs[index], nsk);
318314 free_osk = osk;
319315 err = 0;
....@@ -353,11 +349,15 @@
353349 return 0;
354350 }
355351
352
+static int reuseport_array_map_btf_id;
356353 const struct bpf_map_ops reuseport_array_ops = {
354
+ .map_meta_equal = bpf_map_meta_equal,
357355 .map_alloc_check = reuseport_array_alloc_check,
358356 .map_alloc = reuseport_array_alloc,
359357 .map_free = reuseport_array_free,
360358 .map_lookup_elem = reuseport_array_lookup_elem,
361359 .map_get_next_key = reuseport_array_get_next_key,
362360 .map_delete_elem = reuseport_array_delete_elem,
361
+ .map_btf_name = "reuseport_array",
362
+ .map_btf_id = &reuseport_array_map_btf_id,
363363 };