Skip to content

Commit e59997d

Browse files
author
Alexei Starovoitov
committed
Merge branch 'bpf-arm64-support-exceptions'
Puranjay Mohan says: ==================== bpf, arm64: Support Exceptions Changes in V2->V3: V2: https://lore.kernel.org/all/20230917000045.56377-1-puranjay12@gmail.com/ - Use unwinder from stacktrace.c rather than open coding the unwind logic. - Fix a bug in the prologue related to BPF_FP (Xu Kuohai) Changes in V1->V2: V1: https://lore.kernel.org/all/20230912233942.6734-1-puranjay12@gmail.com/ - Remove exceptions from DENYLIST.aarch64 as they are supported now. The base support for exceptions was merged with [1] and it was enabled for x86-64. This patch set enables the support on ARM64, all sefltests are passing: # ./test_progs -a exceptions #74/1 exceptions/exception_throw_always_1:OK #74/2 exceptions/exception_throw_always_2:OK #74/3 exceptions/exception_throw_unwind_1:OK #74/4 exceptions/exception_throw_unwind_2:OK #74/5 exceptions/exception_throw_default:OK #74/6 exceptions/exception_throw_default_value:OK #74/7 exceptions/exception_tail_call:OK #74/8 exceptions/exception_ext:OK #74/9 exceptions/exception_ext_mod_cb_runtime:OK #74/10 exceptions/exception_throw_subprog:OK #74/11 exceptions/exception_assert_nz_gfunc:OK #74/12 exceptions/exception_assert_zero_gfunc:OK #74/13 exceptions/exception_assert_neg_gfunc:OK #74/14 exceptions/exception_assert_pos_gfunc:OK #74/15 exceptions/exception_assert_negeq_gfunc:OK #74/16 exceptions/exception_assert_poseq_gfunc:OK #74/17 exceptions/exception_assert_nz_gfunc_with:OK #74/18 exceptions/exception_assert_zero_gfunc_with:OK #74/19 exceptions/exception_assert_neg_gfunc_with:OK #74/20 exceptions/exception_assert_pos_gfunc_with:OK #74/21 exceptions/exception_assert_negeq_gfunc_with:OK #74/22 exceptions/exception_assert_poseq_gfunc_with:OK #74/23 exceptions/exception_bad_assert_nz_gfunc:OK #74/24 exceptions/exception_bad_assert_zero_gfunc:OK #74/25 exceptions/exception_bad_assert_neg_gfunc:OK #74/26 exceptions/exception_bad_assert_pos_gfunc:OK #74/27 exceptions/exception_bad_assert_negeq_gfunc:OK #74/28 exceptions/exception_bad_assert_poseq_gfunc:OK #74/29 exceptions/exception_bad_assert_nz_gfunc_with:OK #74/30 exceptions/exception_bad_assert_zero_gfunc_with:OK #74/31 exceptions/exception_bad_assert_neg_gfunc_with:OK #74/32 exceptions/exception_bad_assert_pos_gfunc_with:OK #74/33 exceptions/exception_bad_assert_negeq_gfunc_with:OK #74/34 exceptions/exception_bad_assert_poseq_gfunc_with:OK #74/35 exceptions/exception_assert_range:OK #74/36 exceptions/exception_assert_range_with:OK #74/37 exceptions/exception_bad_assert_range:OK #74/38 exceptions/exception_bad_assert_range_with:OK #74/39 exceptions/non-throwing fentry -> exception_cb:OK #74/40 exceptions/throwing fentry -> exception_cb:OK #74/41 exceptions/non-throwing fexit -> exception_cb:OK #74/42 exceptions/throwing fexit -> exception_cb:OK #74/43 exceptions/throwing extension (with custom cb) -> exception_cb:OK #74/44 exceptions/throwing extension -> global func in exception_cb:OK #74/45 exceptions/exception_ext_mod_cb_runtime:OK #74/46 exceptions/throwing extension (with custom cb) -> global func in exception_cb:OK #74/47 exceptions/exception_ext:OK #74/48 exceptions/non-throwing fentry -> non-throwing subprog:OK #74/49 exceptions/throwing fentry -> non-throwing subprog:OK #74/50 exceptions/non-throwing fentry -> throwing subprog:OK #74/51 exceptions/throwing fentry -> throwing subprog:OK #74/52 exceptions/non-throwing fexit -> non-throwing subprog:OK #74/53 exceptions/throwing fexit -> non-throwing subprog:OK #74/54 exceptions/non-throwing fexit -> throwing subprog:OK #74/55 exceptions/throwing fexit -> throwing subprog:OK #74/56 exceptions/non-throwing fmod_ret -> non-throwing subprog:OK #74/57 exceptions/non-throwing fmod_ret -> non-throwing global subprog:OK #74/58 exceptions/non-throwing extension -> non-throwing subprog:OK #74/59 exceptions/non-throwing extension -> throwing subprog:OK #74/60 exceptions/non-throwing extension -> non-throwing subprog:OK #74/61 exceptions/non-throwing extension -> throwing global subprog:OK #74/62 exceptions/throwing extension -> throwing global subprog:OK #74/63 exceptions/throwing extension -> non-throwing global subprog:OK #74/64 exceptions/non-throwing extension -> main subprog:OK #74/65 exceptions/throwing extension -> main subprog:OK #74/66 exceptions/reject_exception_cb_type_1:OK #74/67 exceptions/reject_exception_cb_type_2:OK #74/68 exceptions/reject_exception_cb_type_3:OK #74/69 exceptions/reject_exception_cb_type_4:OK #74/70 exceptions/reject_async_callback_throw:OK #74/71 exceptions/reject_with_lock:OK #74/72 exceptions/reject_subprog_with_lock:OK #74/73 exceptions/reject_with_rcu_read_lock:OK #74/74 exceptions/reject_subprog_with_rcu_read_lock:OK #74/75 exceptions/reject_with_rbtree_add_throw:OK #74/76 exceptions/reject_with_reference:OK #74/77 exceptions/reject_with_cb_reference:OK #74/78 exceptions/reject_with_cb:OK #74/79 exceptions/reject_with_subprog_reference:OK #74/80 exceptions/reject_throwing_exception_cb:OK #74/81 exceptions/reject_exception_cb_call_global_func:OK #74/82 exceptions/reject_exception_cb_call_static_func:OK #74/83 exceptions/reject_multiple_exception_cb:OK #74/84 exceptions/reject_exception_throw_cb:OK #74/85 exceptions/reject_exception_throw_cb_diff:OK #74/86 exceptions/reject_set_exception_cb_bad_ret1:OK #74/87 exceptions/reject_set_exception_cb_bad_ret2:OK #74/88 exceptions/check_assert_eq_int_min:OK #74/89 exceptions/check_assert_eq_int_max:OK #74/90 exceptions/check_assert_eq_zero:OK #74/91 exceptions/check_assert_eq_llong_min:OK #74/92 exceptions/check_assert_eq_llong_max:OK #74/93 exceptions/check_assert_lt_pos:OK #74/94 exceptions/check_assert_lt_zero:OK #74/95 exceptions/check_assert_lt_neg:OK #74/96 exceptions/check_assert_le_pos:OK #74/97 exceptions/check_assert_le_zero:OK #74/98 exceptions/check_assert_le_neg:OK #74/99 exceptions/check_assert_gt_pos:OK #74/100 exceptions/check_assert_gt_zero:OK #74/101 exceptions/check_assert_gt_neg:OK #74/102 exceptions/check_assert_ge_pos:OK #74/103 exceptions/check_assert_ge_zero:OK #74/104 exceptions/check_assert_ge_neg:OK #74/105 exceptions/check_assert_range_s64:OK #74/106 exceptions/check_assert_range_u64:OK #74/107 exceptions/check_assert_single_range_s64:OK #74/108 exceptions/check_assert_single_range_u64:OK #74/109 exceptions/check_assert_generic:OK #74/110 exceptions/check_assert_with_return:OK #74 exceptions:OK Summary: 1/110 PASSED, 0 SKIPPED, 0 FAILED [1] https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git/commit/?h=for-next&id=ec6f1b4db95b7eedb3fe85f4f14e08fa0e9281c3 ==================== Link: https://lore.kernel.org/r/20240201125225.72796-1-puranjay12@gmail.com Signed-off-by: Alexei Starovoitov <ast@kernel.org>
2 parents 2ab256e + 22fc0e8 commit e59997d

3 files changed

Lines changed: 94 additions & 20 deletions

File tree

‎arch/arm64/kernel/stacktrace.c‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
#include <linux/kernel.h>
88
#include <linux/efi.h>
99
#include <linux/export.h>
10+
#include <linux/filter.h>
1011
#include <linux/ftrace.h>
1112
#include <linux/kprobes.h>
1213
#include <linux/sched.h>
@@ -266,6 +267,31 @@ noinline noinstr void arch_stack_walk(stack_trace_consume_fn consume_entry,
266267
kunwind_stack_walk(arch_kunwind_consume_entry, &data, task, regs);
267268
}
268269

270+
struct bpf_unwind_consume_entry_data {
271+
bool (*consume_entry)(void *cookie, u64 ip, u64 sp, u64 fp);
272+
void *cookie;
273+
};
274+
275+
static bool
276+
arch_bpf_unwind_consume_entry(const struct kunwind_state *state, void *cookie)
277+
{
278+
struct bpf_unwind_consume_entry_data *data = cookie;
279+
280+
return data->consume_entry(data->cookie, state->common.pc, 0,
281+
state->common.fp);
282+
}
283+
284+
noinline noinstr void arch_bpf_stack_walk(bool (*consume_entry)(void *cookie, u64 ip, u64 sp,
285+
u64 fp), void *cookie)
286+
{
287+
struct bpf_unwind_consume_entry_data data = {
288+
.consume_entry = consume_entry,
289+
.cookie = cookie,
290+
};
291+
292+
kunwind_stack_walk(arch_bpf_unwind_consume_entry, &data, current, NULL);
293+
}
294+
269295
static bool dump_backtrace_entry(void *arg, unsigned long where)
270296
{
271297
char *loglvl = arg;

‎arch/arm64/net/bpf_jit_comp.c‎

Lines changed: 68 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -285,7 +285,8 @@ static bool is_lsi_offset(int offset, int scale)
285285
/* Tail call offset to jump into */
286286
#define PROLOGUE_OFFSET (BTI_INSNS + 2 + PAC_INSNS + 8)
287287

288-
static int build_prologue(struct jit_ctx *ctx, bool ebpf_from_cbpf)
288+
static int build_prologue(struct jit_ctx *ctx, bool ebpf_from_cbpf,
289+
bool is_exception_cb)
289290
{
290291
const struct bpf_prog *prog = ctx->prog;
291292
const bool is_main_prog = !bpf_is_subprog(prog);
@@ -333,19 +334,34 @@ static int build_prologue(struct jit_ctx *ctx, bool ebpf_from_cbpf)
333334
emit(A64_MOV(1, A64_R(9), A64_LR), ctx);
334335
emit(A64_NOP, ctx);
335336

336-
/* Sign lr */
337-
if (IS_ENABLED(CONFIG_ARM64_PTR_AUTH_KERNEL))
338-
emit(A64_PACIASP, ctx);
339-
340-
/* Save FP and LR registers to stay align with ARM64 AAPCS */
341-
emit(A64_PUSH(A64_FP, A64_LR, A64_SP), ctx);
342-
emit(A64_MOV(1, A64_FP, A64_SP), ctx);
343-
344-
/* Save callee-saved registers */
345-
emit(A64_PUSH(r6, r7, A64_SP), ctx);
346-
emit(A64_PUSH(r8, r9, A64_SP), ctx);
347-
emit(A64_PUSH(fp, tcc, A64_SP), ctx);
348-
emit(A64_PUSH(fpb, A64_R(28), A64_SP), ctx);
337+
if (!is_exception_cb) {
338+
/* Sign lr */
339+
if (IS_ENABLED(CONFIG_ARM64_PTR_AUTH_KERNEL))
340+
emit(A64_PACIASP, ctx);
341+
/* Save FP and LR registers to stay align with ARM64 AAPCS */
342+
emit(A64_PUSH(A64_FP, A64_LR, A64_SP), ctx);
343+
emit(A64_MOV(1, A64_FP, A64_SP), ctx);
344+
345+
/* Save callee-saved registers */
346+
emit(A64_PUSH(r6, r7, A64_SP), ctx);
347+
emit(A64_PUSH(r8, r9, A64_SP), ctx);
348+
emit(A64_PUSH(fp, tcc, A64_SP), ctx);
349+
emit(A64_PUSH(fpb, A64_R(28), A64_SP), ctx);
350+
} else {
351+
/*
352+
* Exception callback receives FP of Main Program as third
353+
* parameter
354+
*/
355+
emit(A64_MOV(1, A64_FP, A64_R(2)), ctx);
356+
/*
357+
* Main Program already pushed the frame record and the
358+
* callee-saved registers. The exception callback will not push
359+
* anything and re-use the main program's stack.
360+
*
361+
* 10 registers are on the stack
362+
*/
363+
emit(A64_SUB_I(1, A64_SP, A64_FP, 80), ctx);
364+
}
349365

350366
/* Set up BPF prog stack base register */
351367
emit(A64_MOV(1, fp, A64_SP), ctx);
@@ -365,6 +381,20 @@ static int build_prologue(struct jit_ctx *ctx, bool ebpf_from_cbpf)
365381
emit_bti(A64_BTI_J, ctx);
366382
}
367383

384+
/*
385+
* Program acting as exception boundary should save all ARM64
386+
* Callee-saved registers as the exception callback needs to recover
387+
* all ARM64 Callee-saved registers in its epilogue.
388+
*/
389+
if (prog->aux->exception_boundary) {
390+
/*
391+
* As we are pushing two more registers, BPF_FP should be moved
392+
* 16 bytes
393+
*/
394+
emit(A64_SUB_I(1, fp, fp, 16), ctx);
395+
emit(A64_PUSH(A64_R(23), A64_R(24), A64_SP), ctx);
396+
}
397+
368398
emit(A64_SUB_I(1, fpb, fp, ctx->fpb_offset), ctx);
369399

370400
/* Stack must be multiples of 16B */
@@ -653,7 +683,7 @@ static void build_plt(struct jit_ctx *ctx)
653683
plt->target = (u64)&dummy_tramp;
654684
}
655685

656-
static void build_epilogue(struct jit_ctx *ctx)
686+
static void build_epilogue(struct jit_ctx *ctx, bool is_exception_cb)
657687
{
658688
const u8 r0 = bpf2a64[BPF_REG_0];
659689
const u8 r6 = bpf2a64[BPF_REG_6];
@@ -666,6 +696,15 @@ static void build_epilogue(struct jit_ctx *ctx)
666696
/* We're done with BPF stack */
667697
emit(A64_ADD_I(1, A64_SP, A64_SP, ctx->stack_size), ctx);
668698

699+
/*
700+
* Program acting as exception boundary pushes R23 and R24 in addition
701+
* to BPF callee-saved registers. Exception callback uses the boundary
702+
* program's stack frame, so recover these extra registers in the above
703+
* two cases.
704+
*/
705+
if (ctx->prog->aux->exception_boundary || is_exception_cb)
706+
emit(A64_POP(A64_R(23), A64_R(24), A64_SP), ctx);
707+
669708
/* Restore x27 and x28 */
670709
emit(A64_POP(fpb, A64_R(28), A64_SP), ctx);
671710
/* Restore fs (x25) and x26 */
@@ -1575,7 +1614,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
15751614
* BPF line info needs ctx->offset[i] to be the offset of
15761615
* instruction[i] in jited image, so build prologue first.
15771616
*/
1578-
if (build_prologue(&ctx, was_classic)) {
1617+
if (build_prologue(&ctx, was_classic, prog->aux->exception_cb)) {
15791618
prog = orig_prog;
15801619
goto out_off;
15811620
}
@@ -1586,7 +1625,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
15861625
}
15871626

15881627
ctx.epilogue_offset = ctx.idx;
1589-
build_epilogue(&ctx);
1628+
build_epilogue(&ctx, prog->aux->exception_cb);
15901629
build_plt(&ctx);
15911630

15921631
extable_align = __alignof__(struct exception_table_entry);
@@ -1614,15 +1653,15 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
16141653
ctx.idx = 0;
16151654
ctx.exentry_idx = 0;
16161655

1617-
build_prologue(&ctx, was_classic);
1656+
build_prologue(&ctx, was_classic, prog->aux->exception_cb);
16181657

16191658
if (build_body(&ctx, extra_pass)) {
16201659
bpf_jit_binary_free(header);
16211660
prog = orig_prog;
16221661
goto out_off;
16231662
}
16241663

1625-
build_epilogue(&ctx);
1664+
build_epilogue(&ctx, prog->aux->exception_cb);
16261665
build_plt(&ctx);
16271666

16281667
/* 3. Extra pass to validate JITed code. */
@@ -2310,3 +2349,13 @@ bool bpf_jit_supports_ptr_xchg(void)
23102349
{
23112350
return true;
23122351
}
2352+
2353+
bool bpf_jit_supports_exceptions(void)
2354+
{
2355+
/* We unwind through both kernel frames starting from within bpf_throw
2356+
* call and BPF frames. Therefore we require FP unwinder to be enabled
2357+
* to walk kernel frames and reach BPF frames in the stack trace.
2358+
* ARM64 kernel is aways compiled with CONFIG_FRAME_POINTER=y
2359+
*/
2360+
return true;
2361+
}

‎tools/testing/selftests/bpf/DENYLIST.aarch64‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
bpf_cookie/multi_kprobe_attach_api # kprobe_multi_link_api_subtest:FAIL:fentry_raw_skel_load unexpected error: -3
22
bpf_cookie/multi_kprobe_link_api # kprobe_multi_link_api_subtest:FAIL:fentry_raw_skel_load unexpected error: -3
3-
exceptions # JIT does not support calling kfunc bpf_throw: -524
43
fexit_sleep # The test never returns. The remaining tests cannot start.
54
kprobe_multi_bench_attach # needs CONFIG_FPROBE
65
kprobe_multi_test # needs CONFIG_FPROBE

0 commit comments

Comments
 (0)