Skip to content

Commit 93acc0f

Browse files
author
Alexei Starovoitov
committed
Merge branch 'support-kcfi-bpf-on-arm64'
Sami Tolvanen says: ==================== Support kCFI + BPF on arm64 These patches add KCFI types to arm64 BPF JIT output. Puranjay and Maxwell have been working on this for some time now, but I haven't seen any progress since June 2024, so I decided to pick up the latest version[1] posted by Maxwell and fix the few remaining issues I noticed. I confirmed that with these patches applied, I no longer see CFI failures in jitted code when running BPF self-tests on arm64. [1] https://lore.kernel.org/linux-arm-kernel/ptrugmna4xb5o5lo4xislf4rlz7avdmd4pfho5fjwtjj7v422u@iqrwfrbwuxrq/ Sami --- v14: - Rebased to fix a merge conflict. v13: https://lore.kernel.org/bpf/20250722205357.3347626-5-samitolvanen@google.com/ - Added emit_u32_data to fix type hashes on big-endian systems based on Xu's suggestion. v12: https://lore.kernel.org/bpf/20250721202015.3530876-5-samitolvanen@google.com/ - Fixed sparse warnings and 32-bit ARM build errors. v11: https://lore.kernel.org/bpf/20250718223345.1075521-5-samitolvanen@google.com/ - Moved cfi_get_func_hash to a static inline with an ifdef guard. - Picked by Will's Acked-by tags. v10: https://lore.kernel.org/bpf/20250715225733.3921432-5-samitolvanen@google.com/ - Rebased to bpf-next/master again. - Added a patch to moved duplicate type hash variables and helper functions to common CFI code. v9: https://lore.kernel.org/bpf/20250505223437.3722164-4-samitolvanen@google.com/ - Rebased to bpf-next/master to fix merge x86 merge conflicts. - Fixed checkpatch warnings about Co-developed-by tags and including <asm/cfi.h>. - Picked up Tested-by tags. v8: https://lore.kernel.org/bpf/20250310222942.1988975-4-samitolvanen@google.com/ - Changed DEFINE_CFI_TYPE to use .4byte to match __CFI_TYPE. - Changed cfi_get_func_hash() to again use get_kernel_nofault(). - Fixed a panic in bpf_jit_free() by resetting prog->bpf_func before calling bpf_jit_binary_pack_hdr(). --- ==================== Link: https://patch.msgid.link/20250801001004.1859976-5-samitolvanen@google.com Signed-off-by: Alexei Starovoitov <ast@kernel.org>
2 parents 13cb757 + 710618c commit 93acc0f

9 files changed

Lines changed: 113 additions & 125 deletions

File tree

‎arch/arm64/include/asm/cfi.h‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
/* SPDX-License-Identifier: GPL-2.0 */
2+
#ifndef _ASM_ARM64_CFI_H
3+
#define _ASM_ARM64_CFI_H
4+
5+
#define __bpfcall
6+
7+
#endif /* _ASM_ARM64_CFI_H */

‎arch/arm64/net/bpf_jit_comp.c‎

Lines changed: 27 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
#include <linux/arm-smccc.h>
1111
#include <linux/bitfield.h>
1212
#include <linux/bpf.h>
13+
#include <linux/cfi.h>
1314
#include <linux/filter.h>
1415
#include <linux/memory.h>
1516
#include <linux/printk.h>
@@ -114,6 +115,14 @@ static inline void emit(const u32 insn, struct jit_ctx *ctx)
114115
ctx->idx++;
115116
}
116117

118+
static inline void emit_u32_data(const u32 data, struct jit_ctx *ctx)
119+
{
120+
if (ctx->image != NULL && ctx->write)
121+
ctx->image[ctx->idx] = data;
122+
123+
ctx->idx++;
124+
}
125+
117126
static inline void emit_a64_mov_i(const int is64, const int reg,
118127
const s32 val, struct jit_ctx *ctx)
119128
{
@@ -174,6 +183,12 @@ static inline void emit_bti(u32 insn, struct jit_ctx *ctx)
174183
emit(insn, ctx);
175184
}
176185

186+
static inline void emit_kcfi(u32 hash, struct jit_ctx *ctx)
187+
{
188+
if (IS_ENABLED(CONFIG_CFI_CLANG))
189+
emit_u32_data(hash, ctx);
190+
}
191+
177192
/*
178193
* Kernel addresses in the vmalloc space use at most 48 bits, and the
179194
* remaining bits are guaranteed to be 0x1. So we can compose the address
@@ -503,7 +518,6 @@ static int build_prologue(struct jit_ctx *ctx, bool ebpf_from_cbpf)
503518
const u8 arena_vm_base = bpf2a64[ARENA_VM_START];
504519
const u8 priv_sp = bpf2a64[PRIVATE_SP];
505520
void __percpu *priv_stack_ptr;
506-
const int idx0 = ctx->idx;
507521
int cur_offset;
508522

509523
/*
@@ -529,6 +543,9 @@ static int build_prologue(struct jit_ctx *ctx, bool ebpf_from_cbpf)
529543
*
530544
*/
531545

546+
emit_kcfi(is_main_prog ? cfi_bpf_hash : cfi_bpf_subprog_hash, ctx);
547+
const int idx0 = ctx->idx;
548+
532549
/* bpf function may be invoked by 3 instruction types:
533550
* 1. bl, attached via freplace to bpf prog via short jump
534551
* 2. br, attached via freplace to bpf prog via long jump
@@ -2146,9 +2163,9 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
21462163
jit_data->ro_header = ro_header;
21472164
}
21482165

2149-
prog->bpf_func = (void *)ctx.ro_image;
2166+
prog->bpf_func = (void *)ctx.ro_image + cfi_get_offset();
21502167
prog->jited = 1;
2151-
prog->jited_len = prog_size;
2168+
prog->jited_len = prog_size - cfi_get_offset();
21522169

21532170
if (!prog->is_func || extra_pass) {
21542171
int i;
@@ -2527,6 +2544,12 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,
25272544
/* return address locates above FP */
25282545
retaddr_off = stack_size + 8;
25292546

2547+
if (flags & BPF_TRAMP_F_INDIRECT) {
2548+
/*
2549+
* Indirect call for bpf_struct_ops
2550+
*/
2551+
emit_kcfi(cfi_get_func_hash(func_addr), ctx);
2552+
}
25302553
/* bpf trampoline may be invoked by 3 instruction types:
25312554
* 1. bl, attached to bpf prog or kernel function via short jump
25322555
* 2. br, attached to bpf prog or kernel function via long jump
@@ -3045,6 +3068,7 @@ void bpf_jit_free(struct bpf_prog *prog)
30453068
sizeof(jit_data->header->size));
30463069
kfree(jit_data);
30473070
}
3071+
prog->bpf_func -= cfi_get_offset();
30483072
hdr = bpf_jit_binary_pack_hdr(prog);
30493073
bpf_jit_binary_pack_free(hdr, NULL);
30503074
priv_stack_ptr = prog->aux->priv_stack_ptr;

‎arch/riscv/include/asm/cfi.h‎

Lines changed: 0 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -14,27 +14,11 @@ struct pt_regs;
1414
#ifdef CONFIG_CFI_CLANG
1515
enum bug_trap_type handle_cfi_failure(struct pt_regs *regs);
1616
#define __bpfcall
17-
static inline int cfi_get_offset(void)
18-
{
19-
return 4;
20-
}
21-
22-
#define cfi_get_offset cfi_get_offset
23-
extern u32 cfi_bpf_hash;
24-
extern u32 cfi_bpf_subprog_hash;
25-
extern u32 cfi_get_func_hash(void *func);
2617
#else
2718
static inline enum bug_trap_type handle_cfi_failure(struct pt_regs *regs)
2819
{
2920
return BUG_TRAP_TYPE_NONE;
3021
}
31-
32-
#define cfi_bpf_hash 0U
33-
#define cfi_bpf_subprog_hash 0U
34-
static inline u32 cfi_get_func_hash(void *func)
35-
{
36-
return 0;
37-
}
3822
#endif /* CONFIG_CFI_CLANG */
3923

4024
#endif /* _ASM_RISCV_CFI_H */

‎arch/riscv/kernel/cfi.c‎

Lines changed: 0 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -75,56 +75,3 @@ enum bug_trap_type handle_cfi_failure(struct pt_regs *regs)
7575

7676
return report_cfi_failure(regs, regs->epc, &target, type);
7777
}
78-
79-
#ifdef CONFIG_CFI_CLANG
80-
struct bpf_insn;
81-
82-
/* Must match bpf_func_t / DEFINE_BPF_PROG_RUN() */
83-
extern unsigned int __bpf_prog_runX(const void *ctx,
84-
const struct bpf_insn *insn);
85-
86-
/*
87-
* Force a reference to the external symbol so the compiler generates
88-
* __kcfi_typid.
89-
*/
90-
__ADDRESSABLE(__bpf_prog_runX);
91-
92-
/* u32 __ro_after_init cfi_bpf_hash = __kcfi_typeid___bpf_prog_runX; */
93-
asm (
94-
" .pushsection .data..ro_after_init,\"aw\",@progbits \n"
95-
" .type cfi_bpf_hash,@object \n"
96-
" .globl cfi_bpf_hash \n"
97-
" .p2align 2, 0x0 \n"
98-
"cfi_bpf_hash: \n"
99-
" .word __kcfi_typeid___bpf_prog_runX \n"
100-
" .size cfi_bpf_hash, 4 \n"
101-
" .popsection \n"
102-
);
103-
104-
/* Must match bpf_callback_t */
105-
extern u64 __bpf_callback_fn(u64, u64, u64, u64, u64);
106-
107-
__ADDRESSABLE(__bpf_callback_fn);
108-
109-
/* u32 __ro_after_init cfi_bpf_subprog_hash = __kcfi_typeid___bpf_callback_fn; */
110-
asm (
111-
" .pushsection .data..ro_after_init,\"aw\",@progbits \n"
112-
" .type cfi_bpf_subprog_hash,@object \n"
113-
" .globl cfi_bpf_subprog_hash \n"
114-
" .p2align 2, 0x0 \n"
115-
"cfi_bpf_subprog_hash: \n"
116-
" .word __kcfi_typeid___bpf_callback_fn \n"
117-
" .size cfi_bpf_subprog_hash, 4 \n"
118-
" .popsection \n"
119-
);
120-
121-
u32 cfi_get_func_hash(void *func)
122-
{
123-
u32 hash;
124-
125-
if (get_kernel_nofault(hash, func - cfi_get_offset()))
126-
return 0;
127-
128-
return hash;
129-
}
130-
#endif

‎arch/x86/include/asm/cfi.h‎

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -116,8 +116,6 @@ struct pt_regs;
116116
#ifdef CONFIG_CFI_CLANG
117117
enum bug_trap_type handle_cfi_failure(struct pt_regs *regs);
118118
#define __bpfcall
119-
extern u32 cfi_bpf_hash;
120-
extern u32 cfi_bpf_subprog_hash;
121119

122120
static inline int cfi_get_offset(void)
123121
{
@@ -135,6 +133,8 @@ static inline int cfi_get_offset(void)
135133
#define cfi_get_offset cfi_get_offset
136134

137135
extern u32 cfi_get_func_hash(void *func);
136+
#define cfi_get_func_hash cfi_get_func_hash
137+
138138
extern int cfi_get_func_arity(void *func);
139139

140140
#ifdef CONFIG_FINEIBT
@@ -153,12 +153,6 @@ static inline enum bug_trap_type handle_cfi_failure(struct pt_regs *regs)
153153
{
154154
return BUG_TRAP_TYPE_NONE;
155155
}
156-
#define cfi_bpf_hash 0U
157-
#define cfi_bpf_subprog_hash 0U
158-
static inline u32 cfi_get_func_hash(void *func)
159-
{
160-
return 0;
161-
}
162156
static inline int cfi_get_func_arity(void *func)
163157
{
164158
return 0;

‎arch/x86/kernel/alternative.c‎

Lines changed: 0 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -1184,43 +1184,6 @@ bool cfi_bhi __ro_after_init = false;
11841184
#endif
11851185

11861186
#ifdef CONFIG_CFI_CLANG
1187-
struct bpf_insn;
1188-
1189-
/* Must match bpf_func_t / DEFINE_BPF_PROG_RUN() */
1190-
extern unsigned int __bpf_prog_runX(const void *ctx,
1191-
const struct bpf_insn *insn);
1192-
1193-
KCFI_REFERENCE(__bpf_prog_runX);
1194-
1195-
/* u32 __ro_after_init cfi_bpf_hash = __kcfi_typeid___bpf_prog_runX; */
1196-
asm (
1197-
" .pushsection .data..ro_after_init,\"aw\",@progbits \n"
1198-
" .type cfi_bpf_hash,@object \n"
1199-
" .globl cfi_bpf_hash \n"
1200-
" .p2align 2, 0x0 \n"
1201-
"cfi_bpf_hash: \n"
1202-
" .long __kcfi_typeid___bpf_prog_runX \n"
1203-
" .size cfi_bpf_hash, 4 \n"
1204-
" .popsection \n"
1205-
);
1206-
1207-
/* Must match bpf_callback_t */
1208-
extern u64 __bpf_callback_fn(u64, u64, u64, u64, u64);
1209-
1210-
KCFI_REFERENCE(__bpf_callback_fn);
1211-
1212-
/* u32 __ro_after_init cfi_bpf_subprog_hash = __kcfi_typeid___bpf_callback_fn; */
1213-
asm (
1214-
" .pushsection .data..ro_after_init,\"aw\",@progbits \n"
1215-
" .type cfi_bpf_subprog_hash,@object \n"
1216-
" .globl cfi_bpf_subprog_hash \n"
1217-
" .p2align 2, 0x0 \n"
1218-
"cfi_bpf_subprog_hash: \n"
1219-
" .long __kcfi_typeid___bpf_callback_fn \n"
1220-
" .size cfi_bpf_subprog_hash, 4 \n"
1221-
" .popsection \n"
1222-
);
1223-
12241187
u32 cfi_get_func_hash(void *func)
12251188
{
12261189
u32 hash;

‎include/linux/cfi.h‎

Lines changed: 39 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -11,16 +11,9 @@
1111
#include <linux/module.h>
1212
#include <asm/cfi.h>
1313

14+
#ifdef CONFIG_CFI_CLANG
1415
extern bool cfi_warn;
1516

16-
#ifndef cfi_get_offset
17-
static inline int cfi_get_offset(void)
18-
{
19-
return 0;
20-
}
21-
#endif
22-
23-
#ifdef CONFIG_CFI_CLANG
2417
enum bug_trap_type report_cfi_failure(struct pt_regs *regs, unsigned long addr,
2518
unsigned long *target, u32 type);
2619

@@ -29,6 +22,44 @@ static inline enum bug_trap_type report_cfi_failure_noaddr(struct pt_regs *regs,
2922
{
3023
return report_cfi_failure(regs, addr, NULL, 0);
3124
}
25+
26+
#ifndef cfi_get_offset
27+
/*
28+
* Returns the CFI prefix offset. By default, the compiler emits only
29+
* a 4-byte CFI type hash before the function. If an architecture
30+
* uses -fpatchable-function-entry=N,M where M>0 to change the prefix
31+
* offset, they must override this function.
32+
*/
33+
static inline int cfi_get_offset(void)
34+
{
35+
return 4;
36+
}
37+
#endif
38+
39+
#ifndef cfi_get_func_hash
40+
static inline u32 cfi_get_func_hash(void *func)
41+
{
42+
u32 hash;
43+
44+
if (get_kernel_nofault(hash, func - cfi_get_offset()))
45+
return 0;
46+
47+
return hash;
48+
}
49+
#endif
50+
51+
/* CFI type hashes for BPF function types */
52+
extern u32 cfi_bpf_hash;
53+
extern u32 cfi_bpf_subprog_hash;
54+
55+
#else /* CONFIG_CFI_CLANG */
56+
57+
static inline int cfi_get_offset(void) { return 0; }
58+
static inline u32 cfi_get_func_hash(void *func) { return 0; }
59+
60+
#define cfi_bpf_hash 0U
61+
#define cfi_bpf_subprog_hash 0U
62+
3263
#endif /* CONFIG_CFI_CLANG */
3364

3465
#ifdef CONFIG_ARCH_USES_CFI_TRAPS

‎include/linux/cfi_types.h‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,5 +41,28 @@
4141
SYM_TYPED_START(name, SYM_L_GLOBAL, SYM_A_ALIGN)
4242
#endif
4343

44+
#else /* __ASSEMBLY__ */
45+
46+
#ifdef CONFIG_CFI_CLANG
47+
#define DEFINE_CFI_TYPE(name, func) \
48+
/* \
49+
* Force a reference to the function so the compiler generates \
50+
* __kcfi_typeid_<func>. \
51+
*/ \
52+
__ADDRESSABLE(func); \
53+
/* u32 name __ro_after_init = __kcfi_typeid_<func> */ \
54+
extern u32 name; \
55+
asm ( \
56+
" .pushsection .data..ro_after_init,\"aw\",\%progbits \n" \
57+
" .type " #name ",\%object \n" \
58+
" .globl " #name " \n" \
59+
" .p2align 2, 0x0 \n" \
60+
#name ": \n" \
61+
" .4byte __kcfi_typeid_" #func " \n" \
62+
" .size " #name ", 4 \n" \
63+
" .popsection \n" \
64+
);
65+
#endif
66+
4467
#endif /* __ASSEMBLY__ */
4568
#endif /* _LINUX_CFI_TYPES_H */

‎kernel/cfi.c‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@
55
* Copyright (C) 2022 Google LLC
66
*/
77

8+
#include <linux/bpf.h>
9+
#include <linux/cfi_types.h>
810
#include <linux/cfi.h>
911

1012
bool cfi_warn __ro_after_init = IS_ENABLED(CONFIG_CFI_PERMISSIVE);
@@ -27,6 +29,19 @@ enum bug_trap_type report_cfi_failure(struct pt_regs *regs, unsigned long addr,
2729
return BUG_TRAP_TYPE_BUG;
2830
}
2931

32+
/*
33+
* Declare two non-existent functions with types that match bpf_func_t and
34+
* bpf_callback_t pointers, and use DEFINE_CFI_TYPE to define type hash
35+
* variables for each function type. The cfi_bpf_* variables are used by
36+
* arch-specific BPF JIT implementations to ensure indirectly callable JIT
37+
* code has matching CFI type hashes.
38+
*/
39+
extern typeof(*(bpf_func_t)0) __bpf_prog_runX;
40+
DEFINE_CFI_TYPE(cfi_bpf_hash, __bpf_prog_runX);
41+
42+
extern typeof(*(bpf_callback_t)0) __bpf_callback_fn;
43+
DEFINE_CFI_TYPE(cfi_bpf_subprog_hash, __bpf_callback_fn);
44+
3045
#ifdef CONFIG_ARCH_USES_CFI_TRAPS
3146
static inline unsigned long trap_address(s32 *p)
3247
{

0 commit comments

Comments
 (0)