.. | .. |
---|
| 1 | +// SPDX-License-Identifier: GPL-2.0-only |
---|
1 | 2 | /* Copyright (c) 2016 Facebook |
---|
2 | | - * |
---|
3 | | - * This program is free software; you can redistribute it and/or |
---|
4 | | - * modify it under the terms of version 2 of the GNU General Public |
---|
5 | | - * License as published by the Free Software Foundation. |
---|
6 | 3 | */ |
---|
7 | 4 | #define _GNU_SOURCE |
---|
8 | 5 | #include <sched.h> |
---|
.. | .. |
---|
14 | 11 | #include <sys/wait.h> |
---|
15 | 12 | #include <stdlib.h> |
---|
16 | 13 | #include <signal.h> |
---|
17 | | -#include <linux/bpf.h> |
---|
18 | 14 | #include <string.h> |
---|
19 | 15 | #include <time.h> |
---|
20 | 16 | #include <sys/resource.h> |
---|
.. | .. |
---|
22 | 18 | #include <errno.h> |
---|
23 | 19 | |
---|
24 | 20 | #include <bpf/bpf.h> |
---|
25 | | -#include "bpf_load.h" |
---|
| 21 | +#include <bpf/libbpf.h> |
---|
26 | 22 | |
---|
27 | 23 | #define TEST_BIT(t) (1U << (t)) |
---|
28 | 24 | #define MAX_NR_CPUS 1024 |
---|
.. | .. |
---|
64 | 60 | [LRU_HASH_LOOKUP] = "lru_hash_lookup_map", |
---|
65 | 61 | }; |
---|
66 | 62 | |
---|
| 63 | +enum map_idx { |
---|
| 64 | + array_of_lru_hashs_idx, |
---|
| 65 | + hash_map_alloc_idx, |
---|
| 66 | + lru_hash_lookup_idx, |
---|
| 67 | + NR_IDXES, |
---|
| 68 | +}; |
---|
| 69 | + |
---|
| 70 | +static int map_fd[NR_IDXES]; |
---|
| 71 | + |
---|
67 | 72 | static int test_flags = ~0; |
---|
68 | 73 | static uint32_t num_map_entries; |
---|
69 | 74 | static uint32_t inner_lru_hash_size; |
---|
70 | | -static int inner_lru_hash_idx = -1; |
---|
71 | | -static int array_of_lru_hashs_idx = -1; |
---|
72 | | -static int lru_hash_lookup_idx = -1; |
---|
73 | 75 | static int lru_hash_lookup_test_entries = 32; |
---|
74 | 76 | static uint32_t max_cnt = 1000000; |
---|
75 | 77 | |
---|
.. | .. |
---|
125 | 127 | __u64 start_time; |
---|
126 | 128 | int i, ret; |
---|
127 | 129 | |
---|
128 | | - if (test == INNER_LRU_HASH_PREALLOC) { |
---|
| 130 | + if (test == INNER_LRU_HASH_PREALLOC && cpu) { |
---|
| 131 | + /* If CPU is not 0, create inner_lru hash map and insert the fd |
---|
| 132 | + * value into the array_of_lru_hash map. In case of CPU 0, |
---|
| 133 | + * 'inner_lru_hash_map' was statically inserted on the map init |
---|
| 134 | + */ |
---|
129 | 135 | int outer_fd = map_fd[array_of_lru_hashs_idx]; |
---|
130 | 136 | unsigned int mycpu, mynode; |
---|
131 | 137 | |
---|
132 | 138 | assert(cpu < MAX_NR_CPUS); |
---|
133 | 139 | |
---|
134 | | - if (cpu) { |
---|
135 | | - ret = syscall(__NR_getcpu, &mycpu, &mynode, NULL); |
---|
136 | | - assert(!ret); |
---|
| 140 | + ret = syscall(__NR_getcpu, &mycpu, &mynode, NULL); |
---|
| 141 | + assert(!ret); |
---|
137 | 142 | |
---|
138 | | - inner_lru_map_fds[cpu] = |
---|
139 | | - bpf_create_map_node(BPF_MAP_TYPE_LRU_HASH, |
---|
140 | | - test_map_names[INNER_LRU_HASH_PREALLOC], |
---|
141 | | - sizeof(uint32_t), |
---|
142 | | - sizeof(long), |
---|
143 | | - inner_lru_hash_size, 0, |
---|
144 | | - mynode); |
---|
145 | | - if (inner_lru_map_fds[cpu] == -1) { |
---|
146 | | - printf("cannot create BPF_MAP_TYPE_LRU_HASH %s(%d)\n", |
---|
147 | | - strerror(errno), errno); |
---|
148 | | - exit(1); |
---|
149 | | - } |
---|
150 | | - } else { |
---|
151 | | - inner_lru_map_fds[cpu] = map_fd[inner_lru_hash_idx]; |
---|
| 143 | + inner_lru_map_fds[cpu] = |
---|
| 144 | + bpf_create_map_node(BPF_MAP_TYPE_LRU_HASH, |
---|
| 145 | + test_map_names[INNER_LRU_HASH_PREALLOC], |
---|
| 146 | + sizeof(uint32_t), |
---|
| 147 | + sizeof(long), |
---|
| 148 | + inner_lru_hash_size, 0, |
---|
| 149 | + mynode); |
---|
| 150 | + if (inner_lru_map_fds[cpu] == -1) { |
---|
| 151 | + printf("cannot create BPF_MAP_TYPE_LRU_HASH %s(%d)\n", |
---|
| 152 | + strerror(errno), errno); |
---|
| 153 | + exit(1); |
---|
152 | 154 | } |
---|
153 | 155 | |
---|
154 | 156 | ret = bpf_map_update_elem(outer_fd, &cpu, |
---|
.. | .. |
---|
380 | 382 | key->data[1] = rand() & 0xff; |
---|
381 | 383 | key->data[2] = rand() & 0xff; |
---|
382 | 384 | key->data[3] = rand() & 0xff; |
---|
383 | | - r = bpf_map_update_elem(map_fd[6], key, &value, 0); |
---|
| 385 | + r = bpf_map_update_elem(map_fd[hash_map_alloc_idx], |
---|
| 386 | + key, &value, 0); |
---|
384 | 387 | assert(!r); |
---|
385 | 388 | } |
---|
386 | 389 | |
---|
.. | .. |
---|
391 | 394 | key->data[3] = 1; |
---|
392 | 395 | value = 128; |
---|
393 | 396 | |
---|
394 | | - r = bpf_map_update_elem(map_fd[6], key, &value, 0); |
---|
| 397 | + r = bpf_map_update_elem(map_fd[hash_map_alloc_idx], key, &value, 0); |
---|
395 | 398 | assert(!r); |
---|
396 | 399 | } |
---|
397 | 400 | |
---|
398 | | -static void fixup_map(struct bpf_map_data *map, int idx) |
---|
| 401 | +static void fixup_map(struct bpf_object *obj) |
---|
399 | 402 | { |
---|
| 403 | + struct bpf_map *map; |
---|
400 | 404 | int i; |
---|
401 | 405 | |
---|
402 | | - if (!strcmp("inner_lru_hash_map", map->name)) { |
---|
403 | | - inner_lru_hash_idx = idx; |
---|
404 | | - inner_lru_hash_size = map->def.max_entries; |
---|
405 | | - } |
---|
| 406 | + bpf_object__for_each_map(map, obj) { |
---|
| 407 | + const char *name = bpf_map__name(map); |
---|
406 | 408 | |
---|
407 | | - if (!strcmp("array_of_lru_hashs", map->name)) { |
---|
408 | | - if (inner_lru_hash_idx == -1) { |
---|
409 | | - printf("inner_lru_hash_map must be defined before array_of_lru_hashs\n"); |
---|
410 | | - exit(1); |
---|
| 409 | + /* Only change the max_entries for the enabled test(s) */ |
---|
| 410 | + for (i = 0; i < NR_TESTS; i++) { |
---|
| 411 | + if (!strcmp(test_map_names[i], name) && |
---|
| 412 | + (check_test_flags(i))) { |
---|
| 413 | + bpf_map__resize(map, num_map_entries); |
---|
| 414 | + continue; |
---|
| 415 | + } |
---|
411 | 416 | } |
---|
412 | | - map->def.inner_map_idx = inner_lru_hash_idx; |
---|
413 | | - array_of_lru_hashs_idx = idx; |
---|
414 | 417 | } |
---|
415 | | - |
---|
416 | | - if (!strcmp("lru_hash_lookup_map", map->name)) |
---|
417 | | - lru_hash_lookup_idx = idx; |
---|
418 | | - |
---|
419 | | - if (num_map_entries <= 0) |
---|
420 | | - return; |
---|
421 | 418 | |
---|
422 | 419 | inner_lru_hash_size = num_map_entries; |
---|
423 | | - |
---|
424 | | - /* Only change the max_entries for the enabled test(s) */ |
---|
425 | | - for (i = 0; i < NR_TESTS; i++) { |
---|
426 | | - if (!strcmp(test_map_names[i], map->name) && |
---|
427 | | - (check_test_flags(i))) { |
---|
428 | | - map->def.max_entries = num_map_entries; |
---|
429 | | - } |
---|
430 | | - } |
---|
431 | 420 | } |
---|
432 | 421 | |
---|
433 | 422 | int main(int argc, char **argv) |
---|
434 | 423 | { |
---|
435 | 424 | struct rlimit r = {RLIM_INFINITY, RLIM_INFINITY}; |
---|
| 425 | + int nr_cpus = sysconf(_SC_NPROCESSORS_ONLN); |
---|
| 426 | + struct bpf_link *links[8]; |
---|
| 427 | + struct bpf_program *prog; |
---|
| 428 | + struct bpf_object *obj; |
---|
| 429 | + struct bpf_map *map; |
---|
436 | 430 | char filename[256]; |
---|
437 | | - int num_cpu = 8; |
---|
| 431 | + int i = 0; |
---|
438 | 432 | |
---|
439 | | - snprintf(filename, sizeof(filename), "%s_kern.o", argv[0]); |
---|
440 | | - setrlimit(RLIMIT_MEMLOCK, &r); |
---|
| 433 | + if (setrlimit(RLIMIT_MEMLOCK, &r)) { |
---|
| 434 | + perror("setrlimit(RLIMIT_MEMLOCK)"); |
---|
| 435 | + return 1; |
---|
| 436 | + } |
---|
441 | 437 | |
---|
442 | 438 | if (argc > 1) |
---|
443 | 439 | test_flags = atoi(argv[1]) ? : test_flags; |
---|
444 | 440 | |
---|
445 | 441 | if (argc > 2) |
---|
446 | | - num_cpu = atoi(argv[2]) ? : num_cpu; |
---|
| 442 | + nr_cpus = atoi(argv[2]) ? : nr_cpus; |
---|
447 | 443 | |
---|
448 | 444 | if (argc > 3) |
---|
449 | 445 | num_map_entries = atoi(argv[3]); |
---|
.. | .. |
---|
451 | 447 | if (argc > 4) |
---|
452 | 448 | max_cnt = atoi(argv[4]); |
---|
453 | 449 | |
---|
454 | | - if (load_bpf_file_fixup_map(filename, fixup_map)) { |
---|
455 | | - printf("%s", bpf_log_buf); |
---|
456 | | - return 1; |
---|
| 450 | + snprintf(filename, sizeof(filename), "%s_kern.o", argv[0]); |
---|
| 451 | + obj = bpf_object__open_file(filename, NULL); |
---|
| 452 | + if (libbpf_get_error(obj)) { |
---|
| 453 | + fprintf(stderr, "ERROR: opening BPF object file failed\n"); |
---|
| 454 | + return 0; |
---|
| 455 | + } |
---|
| 456 | + |
---|
| 457 | + map = bpf_object__find_map_by_name(obj, "inner_lru_hash_map"); |
---|
| 458 | + if (libbpf_get_error(map)) { |
---|
| 459 | + fprintf(stderr, "ERROR: finding a map in obj file failed\n"); |
---|
| 460 | + goto cleanup; |
---|
| 461 | + } |
---|
| 462 | + |
---|
| 463 | + inner_lru_hash_size = bpf_map__max_entries(map); |
---|
| 464 | + if (!inner_lru_hash_size) { |
---|
| 465 | + fprintf(stderr, "ERROR: failed to get map attribute\n"); |
---|
| 466 | + goto cleanup; |
---|
| 467 | + } |
---|
| 468 | + |
---|
| 469 | + /* resize BPF map prior to loading */ |
---|
| 470 | + if (num_map_entries > 0) |
---|
| 471 | + fixup_map(obj); |
---|
| 472 | + |
---|
| 473 | + /* load BPF program */ |
---|
| 474 | + if (bpf_object__load(obj)) { |
---|
| 475 | + fprintf(stderr, "ERROR: loading BPF object file failed\n"); |
---|
| 476 | + goto cleanup; |
---|
| 477 | + } |
---|
| 478 | + |
---|
| 479 | + map_fd[0] = bpf_object__find_map_fd_by_name(obj, "array_of_lru_hashs"); |
---|
| 480 | + map_fd[1] = bpf_object__find_map_fd_by_name(obj, "hash_map_alloc"); |
---|
| 481 | + map_fd[2] = bpf_object__find_map_fd_by_name(obj, "lru_hash_lookup_map"); |
---|
| 482 | + if (map_fd[0] < 0 || map_fd[1] < 0 || map_fd[2] < 0) { |
---|
| 483 | + fprintf(stderr, "ERROR: finding a map in obj file failed\n"); |
---|
| 484 | + goto cleanup; |
---|
| 485 | + } |
---|
| 486 | + |
---|
| 487 | + bpf_object__for_each_program(prog, obj) { |
---|
| 488 | + links[i] = bpf_program__attach(prog); |
---|
| 489 | + if (libbpf_get_error(links[i])) { |
---|
| 490 | + fprintf(stderr, "ERROR: bpf_program__attach failed\n"); |
---|
| 491 | + links[i] = NULL; |
---|
| 492 | + goto cleanup; |
---|
| 493 | + } |
---|
| 494 | + i++; |
---|
457 | 495 | } |
---|
458 | 496 | |
---|
459 | 497 | fill_lpm_trie(); |
---|
460 | 498 | |
---|
461 | | - run_perf_test(num_cpu); |
---|
| 499 | + run_perf_test(nr_cpus); |
---|
462 | 500 | |
---|
| 501 | +cleanup: |
---|
| 502 | + for (i--; i >= 0; i--) |
---|
| 503 | + bpf_link__destroy(links[i]); |
---|
| 504 | + |
---|
| 505 | + bpf_object__close(obj); |
---|
463 | 506 | return 0; |
---|
464 | 507 | } |
---|