Skip to content

Commit 3c302e1

Browse files
author
Alexei Starovoitov
committed
Merge branch 'x86-cfi-bpf-fix-cfi-vs-ebpf'
Peter Zijlstra says: ==================== x86/cfi,bpf: Fix CFI vs eBPF Hi! What started with the simple observation that bpf_dispatcher_*_func() was broken for calling CFI functions with a __nocfi calling context for FineIBT ended up with a complete BPF wide CFI fixup. With these changes on the BPF selftest suite passes without crashing -- there's still a few failures, but Alexei has graciously offered to look into those. (Alexei, I have presumed your SoB on the very last patch, please update as you see fit) Changes since v2 are numerous but include: - cfi_get_offset() -- as a means to communicate the offset (ast) - 5 new patches fixing various BPF internals to be CFI clean Note: it *might* be possible to merge the bpf_bpf_tcp_ca.c:unsupported_ops[] thing into the CFI stubs, as is get_info will have a NULL stub, unlike the others. --- arch/riscv/include/asm/cfi.h | 3 +- arch/riscv/kernel/cfi.c | 2 +- arch/x86/include/asm/cfi.h | 126 +++++++++++++++++++++++++++++++++++++- arch/x86/kernel/alternative.c | 87 +++++++++++++++++++++++--- arch/x86/kernel/cfi.c | 4 +- arch/x86/net/bpf_jit_comp.c | 134 +++++++++++++++++++++++++++++++++++------ include/asm-generic/Kbuild | 1 + include/linux/bpf.h | 27 ++++++++- include/linux/cfi.h | 12 ++++ kernel/bpf/bpf_struct_ops.c | 16 ++--- kernel/bpf/core.c | 25 ++++++++ kernel/bpf/cpumask.c | 8 ++- kernel/bpf/helpers.c | 18 +++++- net/bpf/bpf_dummy_struct_ops.c | 31 +++++++++- net/bpf/test_run.c | 15 ++++- net/ipv4/bpf_tcp_ca.c | 69 +++++++++++++++++++++ 16 files changed, 528 insertions(+), 50 deletions(-) ==================== Link: https://lore.kernel.org/r/20231215091216.135791411@infradead.org Signed-off-by: Alexei Starovoitov <ast@kernel.org>
2 parents 1467aff + 852486b commit 3c302e1

17 files changed

Lines changed: 533 additions & 50 deletions

File tree

‎arch/riscv/include/asm/cfi.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,9 @@
77
*
88
* Copyright (C) 2023 Google LLC
99
*/
10+
#include <linux/bug.h>
1011

11-
#include <linux/cfi.h>
12+
struct pt_regs;
1213

1314
#ifdef CONFIG_CFI_CLANG
1415
enum bug_trap_type handle_cfi_failure(struct pt_regs *regs);

‎arch/riscv/kernel/cfi.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
*
55
* Copyright (C) 2023 Google LLC
66
*/
7-
#include <asm/cfi.h>
7+
#include <linux/cfi.h>
88
#include <asm/insn.h>
99

1010
/*

‎arch/x86/include/asm/cfi.h‎

Lines changed: 125 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,16 +7,140 @@
77
*
88
* Copyright (C) 2022 Google LLC
99
*/
10+
#include <linux/bug.h>
11+
#include <asm/ibt.h>
1012

11-
#include <linux/cfi.h>
13+
/*
14+
* An overview of the various calling conventions...
15+
*
16+
* Traditional:
17+
*
18+
* foo:
19+
* ... code here ...
20+
* ret
21+
*
22+
* direct caller:
23+
* call foo
24+
*
25+
* indirect caller:
26+
* lea foo(%rip), %r11
27+
* ...
28+
* call *%r11
29+
*
30+
*
31+
* IBT:
32+
*
33+
* foo:
34+
* endbr64
35+
* ... code here ...
36+
* ret
37+
*
38+
* direct caller:
39+
* call foo / call foo+4
40+
*
41+
* indirect caller:
42+
* lea foo(%rip), %r11
43+
* ...
44+
* call *%r11
45+
*
46+
*
47+
* kCFI:
48+
*
49+
* __cfi_foo:
50+
* movl $0x12345678, %eax
51+
* # 11 nops when CONFIG_CALL_PADDING
52+
* foo:
53+
* endbr64 # when IBT
54+
* ... code here ...
55+
* ret
56+
*
57+
* direct call:
58+
* call foo # / call foo+4 when IBT
59+
*
60+
* indirect call:
61+
* lea foo(%rip), %r11
62+
* ...
63+
* movl $(-0x12345678), %r10d
64+
* addl -4(%r11), %r10d # -15 when CONFIG_CALL_PADDING
65+
* jz 1f
66+
* ud2
67+
* 1:call *%r11
68+
*
69+
*
70+
* FineIBT (builds as kCFI + CALL_PADDING + IBT + RETPOLINE and runtime patches into):
71+
*
72+
* __cfi_foo:
73+
* endbr64
74+
* subl 0x12345678, %r10d
75+
* jz foo
76+
* ud2
77+
* nop
78+
* foo:
79+
* osp nop3 # was endbr64
80+
* ... code here ...
81+
* ret
82+
*
83+
* direct caller:
84+
* call foo / call foo+4
85+
*
86+
* indirect caller:
87+
* lea foo(%rip), %r11
88+
* ...
89+
* movl $0x12345678, %r10d
90+
* subl $16, %r11
91+
* nop4
92+
* call *%r11
93+
*
94+
*/
95+
enum cfi_mode {
96+
CFI_DEFAULT, /* FineIBT if hardware has IBT, otherwise kCFI */
97+
CFI_OFF, /* Taditional / IBT depending on .config */
98+
CFI_KCFI, /* Optionally CALL_PADDING, IBT, RETPOLINE */
99+
CFI_FINEIBT, /* see arch/x86/kernel/alternative.c */
100+
};
101+
102+
extern enum cfi_mode cfi_mode;
103+
104+
struct pt_regs;
12105

13106
#ifdef CONFIG_CFI_CLANG
14107
enum bug_trap_type handle_cfi_failure(struct pt_regs *regs);
108+
#define __bpfcall
109+
extern u32 cfi_bpf_hash;
110+
extern u32 cfi_bpf_subprog_hash;
111+
112+
static inline int cfi_get_offset(void)
113+
{
114+
switch (cfi_mode) {
115+
case CFI_FINEIBT:
116+
return 16;
117+
case CFI_KCFI:
118+
if (IS_ENABLED(CONFIG_CALL_PADDING))
119+
return 16;
120+
return 5;
121+
default:
122+
return 0;
123+
}
124+
}
125+
#define cfi_get_offset cfi_get_offset
126+
127+
extern u32 cfi_get_func_hash(void *func);
128+
15129
#else
16130
static inline enum bug_trap_type handle_cfi_failure(struct pt_regs *regs)
17131
{
18132
return BUG_TRAP_TYPE_NONE;
19133
}
134+
#define cfi_bpf_hash 0U
135+
#define cfi_bpf_subprog_hash 0U
136+
static inline u32 cfi_get_func_hash(void *func)
137+
{
138+
return 0;
139+
}
20140
#endif /* CONFIG_CFI_CLANG */
21141

142+
#if HAS_KERNEL_IBT == 1
143+
#define CFI_NOSEAL(x) asm(IBT_NOSEAL(__stringify(x)))
144+
#endif
145+
22146
#endif /* _ASM_X86_CFI_H */

‎arch/x86/kernel/alternative.c‎

Lines changed: 79 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
#include <asm/fixmap.h>
3131
#include <asm/paravirt.h>
3232
#include <asm/asm-prototypes.h>
33+
#include <asm/cfi.h>
3334

3435
int __read_mostly alternatives_patched;
3536

@@ -832,15 +833,82 @@ void __init_or_module apply_seal_endbr(s32 *start, s32 *end) { }
832833
#endif /* CONFIG_X86_KERNEL_IBT */
833834

834835
#ifdef CONFIG_FINEIBT
836+
#define __CFI_DEFAULT CFI_DEFAULT
837+
#elif defined(CONFIG_CFI_CLANG)
838+
#define __CFI_DEFAULT CFI_KCFI
839+
#else
840+
#define __CFI_DEFAULT CFI_OFF
841+
#endif
835842

836-
enum cfi_mode {
837-
CFI_DEFAULT,
838-
CFI_OFF,
839-
CFI_KCFI,
840-
CFI_FINEIBT,
841-
};
843+
enum cfi_mode cfi_mode __ro_after_init = __CFI_DEFAULT;
844+
845+
#ifdef CONFIG_CFI_CLANG
846+
struct bpf_insn;
847+
848+
/* Must match bpf_func_t / DEFINE_BPF_PROG_RUN() */
849+
extern unsigned int __bpf_prog_runX(const void *ctx,
850+
const struct bpf_insn *insn);
851+
852+
/*
853+
* Force a reference to the external symbol so the compiler generates
854+
* __kcfi_typid.
855+
*/
856+
__ADDRESSABLE(__bpf_prog_runX);
857+
858+
/* u32 __ro_after_init cfi_bpf_hash = __kcfi_typeid___bpf_prog_runX; */
859+
asm (
860+
" .pushsection .data..ro_after_init,\"aw\",@progbits \n"
861+
" .type cfi_bpf_hash,@object \n"
862+
" .globl cfi_bpf_hash \n"
863+
" .p2align 2, 0x0 \n"
864+
"cfi_bpf_hash: \n"
865+
" .long __kcfi_typeid___bpf_prog_runX \n"
866+
" .size cfi_bpf_hash, 4 \n"
867+
" .popsection \n"
868+
);
869+
870+
/* Must match bpf_callback_t */
871+
extern u64 __bpf_callback_fn(u64, u64, u64, u64, u64);
872+
873+
__ADDRESSABLE(__bpf_callback_fn);
874+
875+
/* u32 __ro_after_init cfi_bpf_subprog_hash = __kcfi_typeid___bpf_callback_fn; */
876+
asm (
877+
" .pushsection .data..ro_after_init,\"aw\",@progbits \n"
878+
" .type cfi_bpf_subprog_hash,@object \n"
879+
" .globl cfi_bpf_subprog_hash \n"
880+
" .p2align 2, 0x0 \n"
881+
"cfi_bpf_subprog_hash: \n"
882+
" .long __kcfi_typeid___bpf_callback_fn \n"
883+
" .size cfi_bpf_subprog_hash, 4 \n"
884+
" .popsection \n"
885+
);
886+
887+
u32 cfi_get_func_hash(void *func)
888+
{
889+
u32 hash;
890+
891+
func -= cfi_get_offset();
892+
switch (cfi_mode) {
893+
case CFI_FINEIBT:
894+
func += 7;
895+
break;
896+
case CFI_KCFI:
897+
func += 1;
898+
break;
899+
default:
900+
return 0;
901+
}
902+
903+
if (get_kernel_nofault(hash, func))
904+
return 0;
905+
906+
return hash;
907+
}
908+
#endif
909+
910+
#ifdef CONFIG_FINEIBT
842911

843-
static enum cfi_mode cfi_mode __ro_after_init = CFI_DEFAULT;
844912
static bool cfi_rand __ro_after_init = true;
845913
static u32 cfi_seed __ro_after_init;
846914

@@ -1149,8 +1217,11 @@ static void __apply_fineibt(s32 *start_retpoline, s32 *end_retpoline,
11491217
goto err;
11501218

11511219
if (cfi_rand) {
1152-
if (builtin)
1220+
if (builtin) {
11531221
cfi_seed = get_random_u32();
1222+
cfi_bpf_hash = cfi_rehash(cfi_bpf_hash);
1223+
cfi_bpf_subprog_hash = cfi_rehash(cfi_bpf_subprog_hash);
1224+
}
11541225

11551226
ret = cfi_rand_preamble(start_cfi, end_cfi);
11561227
if (ret)

‎arch/x86/kernel/cfi.c‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,10 @@
44
*
55
* Copyright (C) 2022 Google LLC
66
*/
7-
#include <asm/cfi.h>
7+
#include <linux/string.h>
8+
#include <linux/cfi.h>
89
#include <asm/insn.h>
910
#include <asm/insn-eval.h>
10-
#include <linux/string.h>
1111

1212
/*
1313
* Returns the target address and the expected type when regs->ip points

0 commit comments

Comments
 (0)