| .. | .. |
|---|
| 1 | +// SPDX-License-Identifier: GPL-2.0-only |
|---|
| 1 | 2 | /* Copyright (c) 2016 Facebook |
|---|
| 2 | | - * |
|---|
| 3 | | - * This program is free software; you can redistribute it and/or |
|---|
| 4 | | - * modify it under the terms of version 2 of the GNU General Public |
|---|
| 5 | | - * License as published by the Free Software Foundation. |
|---|
| 6 | 3 | */ |
|---|
| 7 | 4 | #define _GNU_SOURCE |
|---|
| 8 | 5 | #include <sched.h> |
|---|
| .. | .. |
|---|
| 14 | 11 | #include <sys/wait.h> |
|---|
| 15 | 12 | #include <stdlib.h> |
|---|
| 16 | 13 | #include <signal.h> |
|---|
| 17 | | -#include <linux/bpf.h> |
|---|
| 18 | 14 | #include <string.h> |
|---|
| 19 | 15 | #include <time.h> |
|---|
| 20 | 16 | #include <sys/resource.h> |
|---|
| .. | .. |
|---|
| 22 | 18 | #include <errno.h> |
|---|
| 23 | 19 | |
|---|
| 24 | 20 | #include <bpf/bpf.h> |
|---|
| 25 | | -#include "bpf_load.h" |
|---|
| 21 | +#include <bpf/libbpf.h> |
|---|
| 26 | 22 | |
|---|
| 27 | 23 | #define TEST_BIT(t) (1U << (t)) |
|---|
| 28 | 24 | #define MAX_NR_CPUS 1024 |
|---|
| .. | .. |
|---|
| 64 | 60 | [LRU_HASH_LOOKUP] = "lru_hash_lookup_map", |
|---|
| 65 | 61 | }; |
|---|
| 66 | 62 | |
|---|
| 63 | +enum map_idx { |
|---|
| 64 | + array_of_lru_hashs_idx, |
|---|
| 65 | + hash_map_alloc_idx, |
|---|
| 66 | + lru_hash_lookup_idx, |
|---|
| 67 | + NR_IDXES, |
|---|
| 68 | +}; |
|---|
| 69 | + |
|---|
| 70 | +static int map_fd[NR_IDXES]; |
|---|
| 71 | + |
|---|
| 67 | 72 | static int test_flags = ~0; |
|---|
| 68 | 73 | static uint32_t num_map_entries; |
|---|
| 69 | 74 | static uint32_t inner_lru_hash_size; |
|---|
| 70 | | -static int inner_lru_hash_idx = -1; |
|---|
| 71 | | -static int array_of_lru_hashs_idx = -1; |
|---|
| 72 | | -static int lru_hash_lookup_idx = -1; |
|---|
| 73 | 75 | static int lru_hash_lookup_test_entries = 32; |
|---|
| 74 | 76 | static uint32_t max_cnt = 1000000; |
|---|
| 75 | 77 | |
|---|
| .. | .. |
|---|
| 125 | 127 | __u64 start_time; |
|---|
| 126 | 128 | int i, ret; |
|---|
| 127 | 129 | |
|---|
| 128 | | - if (test == INNER_LRU_HASH_PREALLOC) { |
|---|
| 130 | + if (test == INNER_LRU_HASH_PREALLOC && cpu) { |
|---|
| 131 | + /* If CPU is not 0, create inner_lru hash map and insert the fd |
|---|
| 132 | + * value into the array_of_lru_hash map. In case of CPU 0, |
|---|
| 133 | + * 'inner_lru_hash_map' was statically inserted on the map init |
|---|
| 134 | + */ |
|---|
| 129 | 135 | int outer_fd = map_fd[array_of_lru_hashs_idx]; |
|---|
| 130 | 136 | unsigned int mycpu, mynode; |
|---|
| 131 | 137 | |
|---|
| 132 | 138 | assert(cpu < MAX_NR_CPUS); |
|---|
| 133 | 139 | |
|---|
| 134 | | - if (cpu) { |
|---|
| 135 | | - ret = syscall(__NR_getcpu, &mycpu, &mynode, NULL); |
|---|
| 136 | | - assert(!ret); |
|---|
| 140 | + ret = syscall(__NR_getcpu, &mycpu, &mynode, NULL); |
|---|
| 141 | + assert(!ret); |
|---|
| 137 | 142 | |
|---|
| 138 | | - inner_lru_map_fds[cpu] = |
|---|
| 139 | | - bpf_create_map_node(BPF_MAP_TYPE_LRU_HASH, |
|---|
| 140 | | - test_map_names[INNER_LRU_HASH_PREALLOC], |
|---|
| 141 | | - sizeof(uint32_t), |
|---|
| 142 | | - sizeof(long), |
|---|
| 143 | | - inner_lru_hash_size, 0, |
|---|
| 144 | | - mynode); |
|---|
| 145 | | - if (inner_lru_map_fds[cpu] == -1) { |
|---|
| 146 | | - printf("cannot create BPF_MAP_TYPE_LRU_HASH %s(%d)\n", |
|---|
| 147 | | - strerror(errno), errno); |
|---|
| 148 | | - exit(1); |
|---|
| 149 | | - } |
|---|
| 150 | | - } else { |
|---|
| 151 | | - inner_lru_map_fds[cpu] = map_fd[inner_lru_hash_idx]; |
|---|
| 143 | + inner_lru_map_fds[cpu] = |
|---|
| 144 | + bpf_create_map_node(BPF_MAP_TYPE_LRU_HASH, |
|---|
| 145 | + test_map_names[INNER_LRU_HASH_PREALLOC], |
|---|
| 146 | + sizeof(uint32_t), |
|---|
| 147 | + sizeof(long), |
|---|
| 148 | + inner_lru_hash_size, 0, |
|---|
| 149 | + mynode); |
|---|
| 150 | + if (inner_lru_map_fds[cpu] == -1) { |
|---|
| 151 | + printf("cannot create BPF_MAP_TYPE_LRU_HASH %s(%d)\n", |
|---|
| 152 | + strerror(errno), errno); |
|---|
| 153 | + exit(1); |
|---|
| 152 | 154 | } |
|---|
| 153 | 155 | |
|---|
| 154 | 156 | ret = bpf_map_update_elem(outer_fd, &cpu, |
|---|
| .. | .. |
|---|
| 380 | 382 | key->data[1] = rand() & 0xff; |
|---|
| 381 | 383 | key->data[2] = rand() & 0xff; |
|---|
| 382 | 384 | key->data[3] = rand() & 0xff; |
|---|
| 383 | | - r = bpf_map_update_elem(map_fd[6], key, &value, 0); |
|---|
| 385 | + r = bpf_map_update_elem(map_fd[hash_map_alloc_idx], |
|---|
| 386 | + key, &value, 0); |
|---|
| 384 | 387 | assert(!r); |
|---|
| 385 | 388 | } |
|---|
| 386 | 389 | |
|---|
| .. | .. |
|---|
| 391 | 394 | key->data[3] = 1; |
|---|
| 392 | 395 | value = 128; |
|---|
| 393 | 396 | |
|---|
| 394 | | - r = bpf_map_update_elem(map_fd[6], key, &value, 0); |
|---|
| 397 | + r = bpf_map_update_elem(map_fd[hash_map_alloc_idx], key, &value, 0); |
|---|
| 395 | 398 | assert(!r); |
|---|
| 396 | 399 | } |
|---|
| 397 | 400 | |
|---|
| 398 | | -static void fixup_map(struct bpf_map_data *map, int idx) |
|---|
| 401 | +static void fixup_map(struct bpf_object *obj) |
|---|
| 399 | 402 | { |
|---|
| 403 | + struct bpf_map *map; |
|---|
| 400 | 404 | int i; |
|---|
| 401 | 405 | |
|---|
| 402 | | - if (!strcmp("inner_lru_hash_map", map->name)) { |
|---|
| 403 | | - inner_lru_hash_idx = idx; |
|---|
| 404 | | - inner_lru_hash_size = map->def.max_entries; |
|---|
| 405 | | - } |
|---|
| 406 | + bpf_object__for_each_map(map, obj) { |
|---|
| 407 | + const char *name = bpf_map__name(map); |
|---|
| 406 | 408 | |
|---|
| 407 | | - if (!strcmp("array_of_lru_hashs", map->name)) { |
|---|
| 408 | | - if (inner_lru_hash_idx == -1) { |
|---|
| 409 | | - printf("inner_lru_hash_map must be defined before array_of_lru_hashs\n"); |
|---|
| 410 | | - exit(1); |
|---|
| 409 | + /* Only change the max_entries for the enabled test(s) */ |
|---|
| 410 | + for (i = 0; i < NR_TESTS; i++) { |
|---|
| 411 | + if (!strcmp(test_map_names[i], name) && |
|---|
| 412 | + (check_test_flags(i))) { |
|---|
| 413 | + bpf_map__resize(map, num_map_entries); |
|---|
| 414 | + continue; |
|---|
| 415 | + } |
|---|
| 411 | 416 | } |
|---|
| 412 | | - map->def.inner_map_idx = inner_lru_hash_idx; |
|---|
| 413 | | - array_of_lru_hashs_idx = idx; |
|---|
| 414 | 417 | } |
|---|
| 415 | | - |
|---|
| 416 | | - if (!strcmp("lru_hash_lookup_map", map->name)) |
|---|
| 417 | | - lru_hash_lookup_idx = idx; |
|---|
| 418 | | - |
|---|
| 419 | | - if (num_map_entries <= 0) |
|---|
| 420 | | - return; |
|---|
| 421 | 418 | |
|---|
| 422 | 419 | inner_lru_hash_size = num_map_entries; |
|---|
| 423 | | - |
|---|
| 424 | | - /* Only change the max_entries for the enabled test(s) */ |
|---|
| 425 | | - for (i = 0; i < NR_TESTS; i++) { |
|---|
| 426 | | - if (!strcmp(test_map_names[i], map->name) && |
|---|
| 427 | | - (check_test_flags(i))) { |
|---|
| 428 | | - map->def.max_entries = num_map_entries; |
|---|
| 429 | | - } |
|---|
| 430 | | - } |
|---|
| 431 | 420 | } |
|---|
| 432 | 421 | |
|---|
| 433 | 422 | int main(int argc, char **argv) |
|---|
| 434 | 423 | { |
|---|
| 435 | 424 | struct rlimit r = {RLIM_INFINITY, RLIM_INFINITY}; |
|---|
| 425 | + int nr_cpus = sysconf(_SC_NPROCESSORS_ONLN); |
|---|
| 426 | + struct bpf_link *links[8]; |
|---|
| 427 | + struct bpf_program *prog; |
|---|
| 428 | + struct bpf_object *obj; |
|---|
| 429 | + struct bpf_map *map; |
|---|
| 436 | 430 | char filename[256]; |
|---|
| 437 | | - int num_cpu = 8; |
|---|
| 431 | + int i = 0; |
|---|
| 438 | 432 | |
|---|
| 439 | | - snprintf(filename, sizeof(filename), "%s_kern.o", argv[0]); |
|---|
| 440 | | - setrlimit(RLIMIT_MEMLOCK, &r); |
|---|
| 433 | + if (setrlimit(RLIMIT_MEMLOCK, &r)) { |
|---|
| 434 | + perror("setrlimit(RLIMIT_MEMLOCK)"); |
|---|
| 435 | + return 1; |
|---|
| 436 | + } |
|---|
| 441 | 437 | |
|---|
| 442 | 438 | if (argc > 1) |
|---|
| 443 | 439 | test_flags = atoi(argv[1]) ? : test_flags; |
|---|
| 444 | 440 | |
|---|
| 445 | 441 | if (argc > 2) |
|---|
| 446 | | - num_cpu = atoi(argv[2]) ? : num_cpu; |
|---|
| 442 | + nr_cpus = atoi(argv[2]) ? : nr_cpus; |
|---|
| 447 | 443 | |
|---|
| 448 | 444 | if (argc > 3) |
|---|
| 449 | 445 | num_map_entries = atoi(argv[3]); |
|---|
| .. | .. |
|---|
| 451 | 447 | if (argc > 4) |
|---|
| 452 | 448 | max_cnt = atoi(argv[4]); |
|---|
| 453 | 449 | |
|---|
| 454 | | - if (load_bpf_file_fixup_map(filename, fixup_map)) { |
|---|
| 455 | | - printf("%s", bpf_log_buf); |
|---|
| 456 | | - return 1; |
|---|
| 450 | + snprintf(filename, sizeof(filename), "%s_kern.o", argv[0]); |
|---|
| 451 | + obj = bpf_object__open_file(filename, NULL); |
|---|
| 452 | + if (libbpf_get_error(obj)) { |
|---|
| 453 | + fprintf(stderr, "ERROR: opening BPF object file failed\n"); |
|---|
| 454 | + return 0; |
|---|
| 455 | + } |
|---|
| 456 | + |
|---|
| 457 | + map = bpf_object__find_map_by_name(obj, "inner_lru_hash_map"); |
|---|
| 458 | + if (libbpf_get_error(map)) { |
|---|
| 459 | + fprintf(stderr, "ERROR: finding a map in obj file failed\n"); |
|---|
| 460 | + goto cleanup; |
|---|
| 461 | + } |
|---|
| 462 | + |
|---|
| 463 | + inner_lru_hash_size = bpf_map__max_entries(map); |
|---|
| 464 | + if (!inner_lru_hash_size) { |
|---|
| 465 | + fprintf(stderr, "ERROR: failed to get map attribute\n"); |
|---|
| 466 | + goto cleanup; |
|---|
| 467 | + } |
|---|
| 468 | + |
|---|
| 469 | + /* resize BPF map prior to loading */ |
|---|
| 470 | + if (num_map_entries > 0) |
|---|
| 471 | + fixup_map(obj); |
|---|
| 472 | + |
|---|
| 473 | + /* load BPF program */ |
|---|
| 474 | + if (bpf_object__load(obj)) { |
|---|
| 475 | + fprintf(stderr, "ERROR: loading BPF object file failed\n"); |
|---|
| 476 | + goto cleanup; |
|---|
| 477 | + } |
|---|
| 478 | + |
|---|
| 479 | + map_fd[0] = bpf_object__find_map_fd_by_name(obj, "array_of_lru_hashs"); |
|---|
| 480 | + map_fd[1] = bpf_object__find_map_fd_by_name(obj, "hash_map_alloc"); |
|---|
| 481 | + map_fd[2] = bpf_object__find_map_fd_by_name(obj, "lru_hash_lookup_map"); |
|---|
| 482 | + if (map_fd[0] < 0 || map_fd[1] < 0 || map_fd[2] < 0) { |
|---|
| 483 | + fprintf(stderr, "ERROR: finding a map in obj file failed\n"); |
|---|
| 484 | + goto cleanup; |
|---|
| 485 | + } |
|---|
| 486 | + |
|---|
| 487 | + bpf_object__for_each_program(prog, obj) { |
|---|
| 488 | + links[i] = bpf_program__attach(prog); |
|---|
| 489 | + if (libbpf_get_error(links[i])) { |
|---|
| 490 | + fprintf(stderr, "ERROR: bpf_program__attach failed\n"); |
|---|
| 491 | + links[i] = NULL; |
|---|
| 492 | + goto cleanup; |
|---|
| 493 | + } |
|---|
| 494 | + i++; |
|---|
| 457 | 495 | } |
|---|
| 458 | 496 | |
|---|
| 459 | 497 | fill_lpm_trie(); |
|---|
| 460 | 498 | |
|---|
| 461 | | - run_perf_test(num_cpu); |
|---|
| 499 | + run_perf_test(nr_cpus); |
|---|
| 462 | 500 | |
|---|
| 501 | +cleanup: |
|---|
| 502 | + for (i--; i >= 0; i--) |
|---|
| 503 | + bpf_link__destroy(links[i]); |
|---|
| 504 | + |
|---|
| 505 | + bpf_object__close(obj); |
|---|
| 463 | 506 | return 0; |
|---|
| 464 | 507 | } |
|---|