Skip to content

Commit c7fbf8d

Browse files
anakryikoanakryiko
committed
Merge branch 'x86-fgraph-bpf-fix-orc-stack-unwind-from-kprobe_multi'
Jiri Olsa says: ==================== x86/fgraph,bpf: Fix ORC stack unwind from kprobe_multi hi, Mahe reported missing function from stack trace on top of kprobe multi program. It turned out the latest fix [1] needs some more fixing. v2 changes: - keep the unwind same as for kprobes, attached function is part of entry probe stacktrace, not kretprobe [Steven] - several change in trigger bench [Andrii] - added selftests for standard kprobes and fentry/fexit probes [Andrii] Note I'll try to add similar stacktrace adjustment for fentry/fexit in separate patchset to not complicate this change. thanks, jirka [1] https://lore.kernel.org/bpf/20251104215405.168643-1-jolsa@kernel.org/ --- ==================== Link: https://patch.msgid.link/20260126211837.472802-1-jolsa@kernel.org Signed-off-by: Andrii Nakryiko <andrii@kernel.org>
2 parents cd77618 + 4173b49 commit c7fbf8d

8 files changed

Lines changed: 190 additions & 16 deletions

File tree

‎arch/x86/include/asm/ftrace.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ arch_ftrace_get_regs(struct ftrace_regs *fregs)
5757
}
5858

5959
#define arch_ftrace_partial_regs(regs) do { \
60-
regs->flags &= ~X86_EFLAGS_FIXED; \
60+
regs->flags |= X86_EFLAGS_FIXED; \
6161
regs->cs = __KERNEL_CS; \
6262
} while (0)
6363

‎arch/x86/kernel/ftrace_64.S‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -364,6 +364,9 @@ SYM_CODE_START(return_to_handler)
364364
UNWIND_HINT_UNDEFINED
365365
ANNOTATE_NOENDBR
366366

367+
/* Store original rsp for pt_regs.sp value. */
368+
movq %rsp, %rdi
369+
367370
/* Restore return_to_handler value that got eaten by previous ret instruction. */
368371
subq $8, %rsp
369372
UNWIND_HINT_FUNC
@@ -374,7 +377,7 @@ SYM_CODE_START(return_to_handler)
374377
movq %rax, RAX(%rsp)
375378
movq %rdx, RDX(%rsp)
376379
movq %rbp, RBP(%rsp)
377-
movq %rsp, RSP(%rsp)
380+
movq %rdi, RSP(%rsp)
378381
movq %rsp, %rdi
379382

380383
call ftrace_return_to_handler

‎tools/testing/selftests/bpf/bench.c‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -265,6 +265,7 @@ static const struct argp_option opts[] = {
265265
{ "verbose", 'v', NULL, 0, "Verbose debug output"},
266266
{ "affinity", 'a', NULL, 0, "Set consumer/producer thread affinity"},
267267
{ "quiet", 'q', NULL, 0, "Be more quiet"},
268+
{ "stacktrace", 's', NULL, 0, "Get stack trace"},
268269
{ "prod-affinity", ARG_PROD_AFFINITY_SET, "CPUSET", 0,
269270
"Set of CPUs for producer threads; implies --affinity"},
270271
{ "cons-affinity", ARG_CONS_AFFINITY_SET, "CPUSET", 0,
@@ -350,6 +351,9 @@ static error_t parse_arg(int key, char *arg, struct argp_state *state)
350351
case 'q':
351352
env.quiet = true;
352353
break;
354+
case 's':
355+
env.stacktrace = true;
356+
break;
353357
case ARG_PROD_AFFINITY_SET:
354358
env.affinity = true;
355359
if (parse_num_list(arg, &env.prod_cpus.cpus,

‎tools/testing/selftests/bpf/bench.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ struct env {
2626
bool list;
2727
bool affinity;
2828
bool quiet;
29+
bool stacktrace;
2930
int consumer_cnt;
3031
int producer_cnt;
3132
int nr_cpus;

‎tools/testing/selftests/bpf/benchs/bench_trigger.c‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,7 @@ static void setup_ctx(void)
146146
bpf_program__set_autoload(ctx.skel->progs.trigger_driver, true);
147147

148148
ctx.skel->rodata->batch_iters = args.batch_iters;
149+
ctx.skel->rodata->stacktrace = env.stacktrace;
149150
}
150151

151152
static void load_ctx(void)

‎tools/testing/selftests/bpf/prog_tests/stacktrace_ips.c‎

Lines changed: 115 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -74,11 +74,20 @@ static void test_stacktrace_ips_kprobe_multi(bool retprobe)
7474

7575
load_kallsyms();
7676

77-
check_stacktrace_ips(bpf_map__fd(skel->maps.stackmap), skel->bss->stack_key, 4,
78-
ksym_get_addr("bpf_testmod_stacktrace_test_3"),
79-
ksym_get_addr("bpf_testmod_stacktrace_test_2"),
80-
ksym_get_addr("bpf_testmod_stacktrace_test_1"),
81-
ksym_get_addr("bpf_testmod_test_read"));
77+
if (retprobe) {
78+
check_stacktrace_ips(bpf_map__fd(skel->maps.stackmap), skel->bss->stack_key, 4,
79+
ksym_get_addr("bpf_testmod_stacktrace_test_3"),
80+
ksym_get_addr("bpf_testmod_stacktrace_test_2"),
81+
ksym_get_addr("bpf_testmod_stacktrace_test_1"),
82+
ksym_get_addr("bpf_testmod_test_read"));
83+
} else {
84+
check_stacktrace_ips(bpf_map__fd(skel->maps.stackmap), skel->bss->stack_key, 5,
85+
ksym_get_addr("bpf_testmod_stacktrace_test"),
86+
ksym_get_addr("bpf_testmod_stacktrace_test_3"),
87+
ksym_get_addr("bpf_testmod_stacktrace_test_2"),
88+
ksym_get_addr("bpf_testmod_stacktrace_test_1"),
89+
ksym_get_addr("bpf_testmod_test_read"));
90+
}
8291

8392
cleanup:
8493
stacktrace_ips__destroy(skel);
@@ -128,6 +137,99 @@ static void test_stacktrace_ips_raw_tp(void)
128137
stacktrace_ips__destroy(skel);
129138
}
130139

140+
static void test_stacktrace_ips_kprobe(bool retprobe)
141+
{
142+
LIBBPF_OPTS(bpf_kprobe_opts, opts,
143+
.retprobe = retprobe
144+
);
145+
LIBBPF_OPTS(bpf_test_run_opts, topts);
146+
struct stacktrace_ips *skel;
147+
148+
skel = stacktrace_ips__open_and_load();
149+
if (!ASSERT_OK_PTR(skel, "stacktrace_ips__open_and_load"))
150+
return;
151+
152+
if (!skel->kconfig->CONFIG_UNWINDER_ORC) {
153+
test__skip();
154+
goto cleanup;
155+
}
156+
157+
skel->links.kprobe_test = bpf_program__attach_kprobe_opts(
158+
skel->progs.kprobe_test,
159+
"bpf_testmod_stacktrace_test", &opts);
160+
if (!ASSERT_OK_PTR(skel->links.kprobe_test, "bpf_program__attach_kprobe_opts"))
161+
goto cleanup;
162+
163+
trigger_module_test_read(1);
164+
165+
load_kallsyms();
166+
167+
if (retprobe) {
168+
check_stacktrace_ips(bpf_map__fd(skel->maps.stackmap), skel->bss->stack_key, 4,
169+
ksym_get_addr("bpf_testmod_stacktrace_test_3"),
170+
ksym_get_addr("bpf_testmod_stacktrace_test_2"),
171+
ksym_get_addr("bpf_testmod_stacktrace_test_1"),
172+
ksym_get_addr("bpf_testmod_test_read"));
173+
} else {
174+
check_stacktrace_ips(bpf_map__fd(skel->maps.stackmap), skel->bss->stack_key, 5,
175+
ksym_get_addr("bpf_testmod_stacktrace_test"),
176+
ksym_get_addr("bpf_testmod_stacktrace_test_3"),
177+
ksym_get_addr("bpf_testmod_stacktrace_test_2"),
178+
ksym_get_addr("bpf_testmod_stacktrace_test_1"),
179+
ksym_get_addr("bpf_testmod_test_read"));
180+
}
181+
182+
cleanup:
183+
stacktrace_ips__destroy(skel);
184+
}
185+
186+
static void test_stacktrace_ips_trampoline(bool retprobe)
187+
{
188+
LIBBPF_OPTS(bpf_test_run_opts, topts);
189+
struct stacktrace_ips *skel;
190+
191+
skel = stacktrace_ips__open_and_load();
192+
if (!ASSERT_OK_PTR(skel, "stacktrace_ips__open_and_load"))
193+
return;
194+
195+
if (!skel->kconfig->CONFIG_UNWINDER_ORC) {
196+
test__skip();
197+
goto cleanup;
198+
}
199+
200+
if (retprobe) {
201+
skel->links.fexit_test = bpf_program__attach_trace(skel->progs.fexit_test);
202+
if (!ASSERT_OK_PTR(skel->links.fexit_test, "bpf_program__attach_trace"))
203+
goto cleanup;
204+
} else {
205+
skel->links.fentry_test = bpf_program__attach_trace(skel->progs.fentry_test);
206+
if (!ASSERT_OK_PTR(skel->links.fentry_test, "bpf_program__attach_trace"))
207+
goto cleanup;
208+
}
209+
210+
trigger_module_test_read(1);
211+
212+
load_kallsyms();
213+
214+
if (retprobe) {
215+
check_stacktrace_ips(bpf_map__fd(skel->maps.stackmap), skel->bss->stack_key, 4,
216+
ksym_get_addr("bpf_testmod_stacktrace_test_3"),
217+
ksym_get_addr("bpf_testmod_stacktrace_test_2"),
218+
ksym_get_addr("bpf_testmod_stacktrace_test_1"),
219+
ksym_get_addr("bpf_testmod_test_read"));
220+
} else {
221+
check_stacktrace_ips(bpf_map__fd(skel->maps.stackmap), skel->bss->stack_key, 5,
222+
ksym_get_addr("bpf_testmod_stacktrace_test"),
223+
ksym_get_addr("bpf_testmod_stacktrace_test_3"),
224+
ksym_get_addr("bpf_testmod_stacktrace_test_2"),
225+
ksym_get_addr("bpf_testmod_stacktrace_test_1"),
226+
ksym_get_addr("bpf_testmod_test_read"));
227+
}
228+
229+
cleanup:
230+
stacktrace_ips__destroy(skel);
231+
}
232+
131233
static void __test_stacktrace_ips(void)
132234
{
133235
if (test__start_subtest("kprobe_multi"))
@@ -136,6 +238,14 @@ static void __test_stacktrace_ips(void)
136238
test_stacktrace_ips_kprobe_multi(true);
137239
if (test__start_subtest("raw_tp"))
138240
test_stacktrace_ips_raw_tp();
241+
if (test__start_subtest("kprobe"))
242+
test_stacktrace_ips_kprobe(false);
243+
if (test__start_subtest("kretprobe"))
244+
test_stacktrace_ips_kprobe(true);
245+
if (test__start_subtest("fentry"))
246+
test_stacktrace_ips_trampoline(false);
247+
if (test__start_subtest("fexit"))
248+
test_stacktrace_ips_trampoline(true);
139249
}
140250
#else
141251
static void __test_stacktrace_ips(void)

‎tools/testing/selftests/bpf/progs/stacktrace_ips.c‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,13 @@ int unused(void)
3131

3232
__u32 stack_key;
3333

34+
SEC("kprobe")
35+
int kprobe_test(struct pt_regs *ctx)
36+
{
37+
stack_key = bpf_get_stackid(ctx, &stackmap, 0);
38+
return 0;
39+
}
40+
3441
SEC("kprobe.multi")
3542
int kprobe_multi_test(struct pt_regs *ctx)
3643
{
@@ -46,4 +53,24 @@ int rawtp_test(void *ctx)
4653
return 0;
4754
}
4855

56+
SEC("fentry/bpf_testmod_stacktrace_test")
57+
int fentry_test(struct pt_regs *ctx)
58+
{
59+
/*
60+
* Skip 2 bpf_program/trampoline stack entries:
61+
* - bpf_prog_bd1f7a949f55fb03_fentry_test
62+
* - bpf_trampoline_182536277701
63+
*/
64+
stack_key = bpf_get_stackid(ctx, &stackmap, 2);
65+
return 0;
66+
}
67+
68+
SEC("fexit/bpf_testmod_stacktrace_test")
69+
int fexit_test(struct pt_regs *ctx)
70+
{
71+
/* Skip 2 bpf_program/trampoline stack entries, check fentry_test. */
72+
stack_key = bpf_get_stackid(ctx, &stackmap, 2);
73+
return 0;
74+
}
75+
4976
char _license[] SEC("license") = "GPL";

‎tools/testing/selftests/bpf/progs/trigger_bench.c‎

Lines changed: 37 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,34 @@ static __always_inline void inc_counter(void)
2525
__sync_add_and_fetch(&hits[cpu & CPU_MASK].value, 1);
2626
}
2727

28+
volatile const int stacktrace;
29+
30+
typedef __u64 stack_trace_t[128];
31+
32+
struct {
33+
__uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
34+
__uint(max_entries, 1);
35+
__type(key, __u32);
36+
__type(value, stack_trace_t);
37+
} stack_heap SEC(".maps");
38+
39+
static __always_inline void do_stacktrace(void *ctx)
40+
{
41+
if (!stacktrace)
42+
return;
43+
44+
__u64 *ptr = bpf_map_lookup_elem(&stack_heap, &(__u32){0});
45+
46+
if (ptr)
47+
bpf_get_stack(ctx, ptr, sizeof(stack_trace_t), 0);
48+
}
49+
50+
static __always_inline void handle(void *ctx)
51+
{
52+
inc_counter();
53+
do_stacktrace(ctx);
54+
}
55+
2856
SEC("?uprobe")
2957
int bench_trigger_uprobe(void *ctx)
3058
{
@@ -81,21 +109,21 @@ int trigger_driver_kfunc(void *ctx)
81109
SEC("?kprobe/bpf_get_numa_node_id")
82110
int bench_trigger_kprobe(void *ctx)
83111
{
84-
inc_counter();
112+
handle(ctx);
85113
return 0;
86114
}
87115

88116
SEC("?kretprobe/bpf_get_numa_node_id")
89117
int bench_trigger_kretprobe(void *ctx)
90118
{
91-
inc_counter();
119+
handle(ctx);
92120
return 0;
93121
}
94122

95123
SEC("?kprobe.multi/bpf_get_numa_node_id")
96124
int bench_trigger_kprobe_multi(void *ctx)
97125
{
98-
inc_counter();
126+
handle(ctx);
99127
return 0;
100128
}
101129

@@ -108,7 +136,7 @@ int bench_kprobe_multi_empty(void *ctx)
108136
SEC("?kretprobe.multi/bpf_get_numa_node_id")
109137
int bench_trigger_kretprobe_multi(void *ctx)
110138
{
111-
inc_counter();
139+
handle(ctx);
112140
return 0;
113141
}
114142

@@ -121,34 +149,34 @@ int bench_kretprobe_multi_empty(void *ctx)
121149
SEC("?fentry/bpf_get_numa_node_id")
122150
int bench_trigger_fentry(void *ctx)
123151
{
124-
inc_counter();
152+
handle(ctx);
125153
return 0;
126154
}
127155

128156
SEC("?fexit/bpf_get_numa_node_id")
129157
int bench_trigger_fexit(void *ctx)
130158
{
131-
inc_counter();
159+
handle(ctx);
132160
return 0;
133161
}
134162

135163
SEC("?fmod_ret/bpf_modify_return_test_tp")
136164
int bench_trigger_fmodret(void *ctx)
137165
{
138-
inc_counter();
166+
handle(ctx);
139167
return -22;
140168
}
141169

142170
SEC("?tp/bpf_test_run/bpf_trigger_tp")
143171
int bench_trigger_tp(void *ctx)
144172
{
145-
inc_counter();
173+
handle(ctx);
146174
return 0;
147175
}
148176

149177
SEC("?raw_tp/bpf_trigger_tp")
150178
int bench_trigger_rawtp(void *ctx)
151179
{
152-
inc_counter();
180+
handle(ctx);
153181
return 0;
154182
}

0 commit comments

Comments
 (0)