Skip to content

Commit 9f2c156

Browse files
Alexei Starovoitovanakryiko
Alexei Starovoitov
authored and
anakryiko
committed
selftests/bpf: Add bpf_arena_list test.
bpf_arena_alloc.h - implements page_frag allocator as a bpf program. bpf_arena_list.h - doubly linked link list as a bpf program. Compiled as a bpf program and as native C code. Signed-off-by: Alexei Starovoitov <ast@kernel.org> Signed-off-by: Andrii Nakryiko <andrii@kernel.org> Link: https://lore.kernel.org/bpf/20240308010812.89848-14-alexei.starovoitov@gmail.com
1 parent 80a4129 commit 9f2c156

4 files changed

Lines changed: 314 additions & 0 deletions

File tree

‎tools/testing/selftests/bpf/bpf_arena_alloc.h‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
2+
/* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */
3+
#pragma once
4+
#include "bpf_arena_common.h"
5+
6+
#ifndef __round_mask
7+
#define __round_mask(x, y) ((__typeof__(x))((y)-1))
8+
#endif
9+
#ifndef round_up
10+
#define round_up(x, y) ((((x)-1) | __round_mask(x, y))+1)
11+
#endif
12+
13+
#ifdef __BPF__
14+
#define NR_CPUS (sizeof(struct cpumask) * 8)
15+
16+
static void __arena * __arena page_frag_cur_page[NR_CPUS];
17+
static int __arena page_frag_cur_offset[NR_CPUS];
18+
19+
/* Simple page_frag allocator */
20+
static inline void __arena* bpf_alloc(unsigned int size)
21+
{
22+
__u64 __arena *obj_cnt;
23+
__u32 cpu = bpf_get_smp_processor_id();
24+
void __arena *page = page_frag_cur_page[cpu];
25+
int __arena *cur_offset = &page_frag_cur_offset[cpu];
26+
int offset;
27+
28+
size = round_up(size, 8);
29+
if (size >= PAGE_SIZE - 8)
30+
return NULL;
31+
if (!page) {
32+
refill:
33+
page = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0);
34+
if (!page)
35+
return NULL;
36+
cast_kern(page);
37+
page_frag_cur_page[cpu] = page;
38+
*cur_offset = PAGE_SIZE - 8;
39+
obj_cnt = page + PAGE_SIZE - 8;
40+
*obj_cnt = 0;
41+
} else {
42+
cast_kern(page);
43+
obj_cnt = page + PAGE_SIZE - 8;
44+
}
45+
46+
offset = *cur_offset - size;
47+
if (offset < 0)
48+
goto refill;
49+
50+
(*obj_cnt)++;
51+
*cur_offset = offset;
52+
return page + offset;
53+
}
54+
55+
static inline void bpf_free(void __arena *addr)
56+
{
57+
__u64 __arena *obj_cnt;
58+
59+
addr = (void __arena *)(((long)addr) & ~(PAGE_SIZE - 1));
60+
obj_cnt = addr + PAGE_SIZE - 8;
61+
if (--(*obj_cnt) == 0)
62+
bpf_arena_free_pages(&arena, addr, 1);
63+
}
64+
#else
65+
static inline void __arena* bpf_alloc(unsigned int size) { return NULL; }
66+
static inline void bpf_free(void __arena *addr) {}
67+
#endif

‎tools/testing/selftests/bpf/bpf_arena_list.h‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
2+
/* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */
3+
#pragma once
4+
#include "bpf_arena_common.h"
5+
6+
struct arena_list_node;
7+
8+
typedef struct arena_list_node __arena arena_list_node_t;
9+
10+
struct arena_list_node {
11+
arena_list_node_t *next;
12+
arena_list_node_t * __arena *pprev;
13+
};
14+
15+
struct arena_list_head {
16+
struct arena_list_node __arena *first;
17+
};
18+
typedef struct arena_list_head __arena arena_list_head_t;
19+
20+
#define list_entry(ptr, type, member) arena_container_of(ptr, type, member)
21+
22+
#define list_entry_safe(ptr, type, member) \
23+
({ typeof(*ptr) * ___ptr = (ptr); \
24+
___ptr ? ({ cast_kern(___ptr); list_entry(___ptr, type, member); }) : NULL; \
25+
})
26+
27+
#ifndef __BPF__
28+
static inline void *bpf_iter_num_new(struct bpf_iter_num *it, int i, int j) { return NULL; }
29+
static inline void bpf_iter_num_destroy(struct bpf_iter_num *it) {}
30+
static inline bool bpf_iter_num_next(struct bpf_iter_num *it) { return true; }
31+
#define cond_break ({})
32+
#endif
33+
34+
/* Safely walk link list elements. Deletion of elements is allowed. */
35+
#define list_for_each_entry(pos, head, member) \
36+
for (void * ___tmp = (pos = list_entry_safe((head)->first, \
37+
typeof(*(pos)), member), \
38+
(void *)0); \
39+
pos && ({ ___tmp = (void *)pos->member.next; 1; }); \
40+
cond_break, \
41+
pos = list_entry_safe((void __arena *)___tmp, typeof(*(pos)), member))
42+
43+
static inline void list_add_head(arena_list_node_t *n, arena_list_head_t *h)
44+
{
45+
arena_list_node_t *first = h->first, * __arena *tmp;
46+
47+
cast_user(first);
48+
cast_kern(n);
49+
WRITE_ONCE(n->next, first);
50+
cast_kern(first);
51+
if (first) {
52+
tmp = &n->next;
53+
cast_user(tmp);
54+
WRITE_ONCE(first->pprev, tmp);
55+
}
56+
cast_user(n);
57+
WRITE_ONCE(h->first, n);
58+
59+
tmp = &h->first;
60+
cast_user(tmp);
61+
cast_kern(n);
62+
WRITE_ONCE(n->pprev, tmp);
63+
}
64+
65+
static inline void __list_del(arena_list_node_t *n)
66+
{
67+
arena_list_node_t *next = n->next, *tmp;
68+
arena_list_node_t * __arena *pprev = n->pprev;
69+
70+
cast_user(next);
71+
cast_kern(pprev);
72+
tmp = *pprev;
73+
cast_kern(tmp);
74+
WRITE_ONCE(tmp, next);
75+
if (next) {
76+
cast_user(pprev);
77+
cast_kern(next);
78+
WRITE_ONCE(next->pprev, pprev);
79+
}
80+
}
81+
82+
#define POISON_POINTER_DELTA 0
83+
84+
#define LIST_POISON1 ((void __arena *) 0x100 + POISON_POINTER_DELTA)
85+
#define LIST_POISON2 ((void __arena *) 0x122 + POISON_POINTER_DELTA)
86+
87+
static inline void list_del(arena_list_node_t *n)
88+
{
89+
__list_del(n);
90+
n->next = LIST_POISON1;
91+
n->pprev = LIST_POISON2;
92+
}

‎tools/testing/selftests/bpf/prog_tests/arena_list.c‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
// SPDX-License-Identifier: GPL-2.0
2+
/* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */
3+
#include <test_progs.h>
4+
#include <sys/mman.h>
5+
#include <network_helpers.h>
6+
7+
#define PAGE_SIZE 4096
8+
9+
#include "bpf_arena_list.h"
10+
#include "arena_list.skel.h"
11+
12+
struct elem {
13+
struct arena_list_node node;
14+
__u64 value;
15+
};
16+
17+
static int list_sum(struct arena_list_head *head)
18+
{
19+
struct elem __arena *n;
20+
int sum = 0;
21+
22+
list_for_each_entry(n, head, node)
23+
sum += n->value;
24+
return sum;
25+
}
26+
27+
static void test_arena_list_add_del(int cnt)
28+
{
29+
LIBBPF_OPTS(bpf_test_run_opts, opts);
30+
struct arena_list *skel;
31+
int expected_sum = (u64)cnt * (cnt - 1) / 2;
32+
int ret, sum;
33+
34+
skel = arena_list__open_and_load();
35+
if (!ASSERT_OK_PTR(skel, "arena_list__open_and_load"))
36+
return;
37+
38+
skel->bss->cnt = cnt;
39+
ret = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.arena_list_add), &opts);
40+
ASSERT_OK(ret, "ret_add");
41+
ASSERT_OK(opts.retval, "retval");
42+
if (skel->bss->skip) {
43+
printf("%s:SKIP:compiler doesn't support arena_cast\n", __func__);
44+
test__skip();
45+
goto out;
46+
}
47+
sum = list_sum(skel->bss->list_head);
48+
ASSERT_EQ(sum, expected_sum, "sum of elems");
49+
ASSERT_EQ(skel->arena->arena_sum, expected_sum, "__arena sum of elems");
50+
ASSERT_EQ(skel->arena->test_val, cnt + 1, "num of elems");
51+
52+
ret = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.arena_list_del), &opts);
53+
ASSERT_OK(ret, "ret_del");
54+
sum = list_sum(skel->bss->list_head);
55+
ASSERT_EQ(sum, 0, "sum of list elems after del");
56+
ASSERT_EQ(skel->bss->list_sum, expected_sum, "sum of list elems computed by prog");
57+
ASSERT_EQ(skel->arena->arena_sum, expected_sum, "__arena sum of elems");
58+
out:
59+
arena_list__destroy(skel);
60+
}
61+
62+
void test_arena_list(void)
63+
{
64+
if (test__start_subtest("arena_list_1"))
65+
test_arena_list_add_del(1);
66+
if (test__start_subtest("arena_list_1000"))
67+
test_arena_list_add_del(1000);
68+
}

‎tools/testing/selftests/bpf/progs/arena_list.c‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
// SPDX-License-Identifier: GPL-2.0
2+
/* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */
3+
#include <vmlinux.h>
4+
#include <bpf/bpf_helpers.h>
5+
#include <bpf/bpf_tracing.h>
6+
#include <bpf/bpf_core_read.h>
7+
#include "bpf_experimental.h"
8+
9+
struct {
10+
__uint(type, BPF_MAP_TYPE_ARENA);
11+
__uint(map_flags, BPF_F_MMAPABLE);
12+
__uint(max_entries, 100); /* number of pages */
13+
#ifdef __TARGET_ARCH_arm64
14+
__ulong(map_extra, 0x1ull << 32); /* start of mmap() region */
15+
#else
16+
__ulong(map_extra, 0x1ull << 44); /* start of mmap() region */
17+
#endif
18+
} arena SEC(".maps");
19+
20+
#include "bpf_arena_alloc.h"
21+
#include "bpf_arena_list.h"
22+
23+
struct elem {
24+
struct arena_list_node node;
25+
__u64 value;
26+
};
27+
28+
struct arena_list_head __arena *list_head;
29+
int list_sum;
30+
int cnt;
31+
bool skip = false;
32+
33+
#ifdef __BPF_FEATURE_ARENA_CAST
34+
long __arena arena_sum;
35+
int __arena test_val = 1;
36+
struct arena_list_head __arena global_head;
37+
#else
38+
long arena_sum SEC(".arena.1");
39+
int test_val SEC(".arena.1");
40+
#endif
41+
42+
int zero;
43+
44+
SEC("syscall")
45+
int arena_list_add(void *ctx)
46+
{
47+
#ifdef __BPF_FEATURE_ARENA_CAST
48+
__u64 i;
49+
50+
list_head = &global_head;
51+
52+
for (i = zero; i < cnt; cond_break, i++) {
53+
struct elem __arena *n = bpf_alloc(sizeof(*n));
54+
55+
test_val++;
56+
n->value = i;
57+
arena_sum += i;
58+
list_add_head(&n->node, list_head);
59+
}
60+
#else
61+
skip = true;
62+
#endif
63+
return 0;
64+
}
65+
66+
SEC("syscall")
67+
int arena_list_del(void *ctx)
68+
{
69+
#ifdef __BPF_FEATURE_ARENA_CAST
70+
struct elem __arena *n;
71+
int sum = 0;
72+
73+
arena_sum = 0;
74+
list_for_each_entry(n, list_head, node) {
75+
sum += n->value;
76+
arena_sum += n->value;
77+
list_del(&n->node);
78+
bpf_free(n);
79+
}
80+
list_sum = sum;
81+
#else
82+
skip = true;
83+
#endif
84+
return 0;
85+
}
86+
87+
char _license[] SEC("license") = "GPL";

0 commit comments

Comments
 (0)